[ Nov 14 04:42:23 Enabled. ] [ Nov 14 04:42:23 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Nov 14 04:42:23 Method "start" exited with status 0. ] [ Nov 14 04:42:23 Rereading configuration. ] [ Nov 14 04:42:23 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.325054715Z","hostname":"sock","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.325219639Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.378930175Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.378968566Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.492757801Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.501394581Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:23.509781784Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.112822977Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.251518344Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.251554595Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.25172394Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.251854594Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.251863554Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.251877154Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254724806Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:24.254731847Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254743177Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254758407Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254781228Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254815689Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254918932Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254939823Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.254947773Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.26589538Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.265923041Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.36534862Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835725752Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835758543Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835764883Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835780053Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835790213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835801524Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.835806634Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.844275689Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.84429708Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:24.93476925Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409855137Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409898808Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409933599Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409940279Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.40995132Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.40995951Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409988671Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.409998681Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410010781Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410014661Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410019252Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410027152Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410036722Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410071783Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410077173Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410093094Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410098144Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410102154Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410105264Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410108504Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410122275Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410129905Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410133545Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410137685Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410142715Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410146475Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410163186Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410168426Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410172046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410175446Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410181136Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410187057Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410204357Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410207977Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410248008Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410256099Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.41029896Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410325861Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410330831Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410335261Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410361631Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410366252Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410369432Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410373992Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410378182Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410382152Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410415243Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410420283Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410423903Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410427924Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410431134Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410434804Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410458324Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410462654Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410468765Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410478445Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410482005Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410504446Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410509836Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410513296Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410516616Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410576558Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410590398Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410598638Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410613679Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410618089Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.410622009Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.41065376Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.41066288Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.42171347Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:736"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.421748461Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999919198s","next_cleanup":"Instant { tv_sec: 1324, tv_nsec: 26596473 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.421789682Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.421823743Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.421885555Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.424508141Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.424900373Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.426562731Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.4272362Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1198"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.42724276Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427309092Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427347223Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427363784Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427369644Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1261"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427390845Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427400155Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427409245Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427417925Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1303"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427421456Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427424996Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427448826Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427463267Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.427759235Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: f971eb12-0fea-4084-96d3-74198bddd289,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: c073d51c-abaa-48df-abb6-2f053e9c1bec (sled),\n rack_id: f971eb12-0fea-4084-96d3-74198bddd289,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428003002Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428007883Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1337"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428046104Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428055104Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428220169Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428338292Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428392824Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c073d51c-abaa-48df-abb6-2f053e9c1bec (sled), rack_id: f971eb12-0fea-4084-96d3-74198bddd289, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.42860335Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4244"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.428672192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4644"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.433416949Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.439765663Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.439932488Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444216872Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444276684Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444370227Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:35341","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: c073d51c-abaa-48df-abb6-2f053e9c1bec (sled), rack_id: f971eb12-0fea-4084-96d3-74198bddd289, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444510651Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444520791Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444558522Z","hostname":"sock","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444570862Z","hostname":"sock","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444577742Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444584183Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:809"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444629464Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444635964Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444646984Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444668145Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:798"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444672835Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:826"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444678675Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:592"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444695476Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.444703586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:697","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.460433872Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.481136501Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.498308778Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.508783962Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:408"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.508823483Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.508828823Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:420"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.508850334Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 83.17074ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.511849741Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f50cb415-0dfc-459e-ba73-f13e2715c662\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f50cb415-0dfc-459e-ba73-f13e2715c662\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_8036e7f6-3d88-4263-96f1-ac55d658c5c4/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.523030314Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_8036e7f6-3d88-4263-96f1-ac55d658c5c4/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.54636291Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_8036e7f6-3d88-4263-96f1-ac55d658c5c4/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.546404631Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.551279012Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:435"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.551289383Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_8036e7f6-3d88-4263-96f1-ac55d658c5c4/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.551294213Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.596464041Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 173.474498ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.599113228Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b9c56f2-473e-4cef-b043-7511c2ce69d3\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b9c56f2-473e-4cef-b043-7511c2ce69d3\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.634488792Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.77462859Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 193.361278ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.775281869Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c44638c3-9cf5-4b72-af14-6103a89d204d\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c44638c3-9cf5-4b72-af14-6103a89d204d\" }"} {"msg":"Mounting oxi_8036e7f6-3d88-4263-96f1-ac55d658c5c4/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.852714112Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.865180363Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.865261705Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.865286756Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.865301786Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.974736014Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 157.71967ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:25.976242338Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a25fe9f7-ff7f-42bb-ad00-dfeb9cc17ea9\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a25fe9f7-ff7f-42bb-ad00-dfeb9cc17ea9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.135297434Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 167.903097ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.136024585Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66d3ec07-0452-4550-b8c4-870216ad5e8a\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"66d3ec07-0452-4550-b8c4-870216ad5e8a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.305089831Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 290.94105ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.30572915Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91bbaf01-8ef1-4672-955d-e1ad5118aaff\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:42:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91bbaf01-8ef1-4672-955d-e1ad5118aaff\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.59647677Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.599869658Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.602509204Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:42027","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.602542165Z","hostname":"sock","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.602594227Z","hostname":"sock","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:42027","producer_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:26.603627757Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"132.791272ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.604464541Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.604470951Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.604511562Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:920","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:26.604517152Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:26.738185983Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"627.772368ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:27.3679731Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.072235736s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:28.441736494Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.485786979s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.148462039Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2668"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.14848774Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2913"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.14849338Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2538"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.148497591Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2585"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.148514841Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2994"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.148522741Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3039"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.150441987Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2838"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.151657852Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2579"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.151786356Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:29.15192577Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 33.546932ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:29.506985772Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 148.533306ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:29.720560967Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 250.282414ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:29.908812428Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:29.928799407Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.904290243s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 485.063226ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:30.295479236Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.08396061s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:31.014801806Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.385228064s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:32.242765496Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:32.83477839Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.64790977s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 631.42414ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:33.684126976Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 873.005336ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:34.333373576Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:35.49448Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.524083437Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.524438517Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525383005Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525407525Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525416166Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525424366Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525609881Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525808627Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525841828Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525851758Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.525886709Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:35341","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699229069Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699344962Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699480276Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699554579Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699566509Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.69959338Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.6996099Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699638111Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.699652421Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.703963766Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4268","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.703983887Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4284"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.703990517Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.988510845Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33410"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.988847905Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.988858085Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.988915447Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.988929027Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:35.98903065Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"924b6aec-c748-42f2-9b99-7ca2a3c9f0c8","remote_addr":"[fd00:1122:3344:101::1]:33410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":294,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.018911216Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.019105651Z","hostname":"sock","pid":1572,"uri":"/sled-role","method":"GET","req_id":"72ed7506-8149-4fd0-9bd3-929ffa877732","remote_addr":"[fd00:1122:3344:101::1]:63294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.019829402Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.019877284Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.020378828Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.020764099Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:980"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: e7f73cb3-2603-4cca-ac59-aacb217af155, pool_id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e2c449ea-97cc-41fd-9068-f2d1755e118e, pool_id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5ef2a600-e0c1-45f9-ac00-dd04915e4c6a, pool_id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 5ad41b7d-910e-49cb-98f1-d37de302f53f, pool_id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 6eb1eb53-a913-4c05-84d6-08c552bfadf3, pool_id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 1f046e34-b8ec-45e0-9c34-86f8757336c1, pool_id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 6fcce26e-6bff-4122-800d-46b6729919a6, pool_id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 1c8a4219-f71e-4ed7-9c86-d8648bcd5585, pool_id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 96afba8e-7682-4c33-91d3-94cf75d8ba67, pool_id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050108579Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:382"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050192812Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60559"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.05049024Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:884"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: e7f73cb3-2603-4cca-ac59-aacb217af155, pool_id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e2c449ea-97cc-41fd-9068-f2d1755e118e, pool_id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5ef2a600-e0c1-45f9-ac00-dd04915e4c6a, pool_id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 5ad41b7d-910e-49cb-98f1-d37de302f53f, pool_id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 6eb1eb53-a913-4c05-84d6-08c552bfadf3, pool_id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 1f046e34-b8ec-45e0-9c34-86f8757336c1, pool_id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 6fcce26e-6bff-4122-800d-46b6729919a6, pool_id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 1c8a4219-f71e-4ed7-9c86-d8648bcd5585, pool_id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 96afba8e-7682-4c33-91d3-94cf75d8ba67, pool_id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050529151Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050538371Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050547912Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050552022Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1171"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050568632Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050572883Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050577633Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.050581383Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.193604985Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.193635566Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b55e819d-82d6-47a9-8489-82b572951fbc (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:36.620883498Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:36.621059473Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13863341701885100570"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:36.621162556Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14763723751087087726"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.825113102Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.874329728Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882277218Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.8823644Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882371771Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882381601Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882392531Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882497004Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.882502304Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.916169169Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:36.924574083Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_b55e819d-82d6-47a9-8489-82b572951fbc","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.095051879Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.101951429Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.101982959Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.10200116Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.10200672Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.10201291Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.112131193Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.112164134Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.206813895Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214646612Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214691683Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214698424Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214736575Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214746925Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214865668Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.214872799Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.248448631Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.256111023Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.429418492Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.437100224Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.437126015Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.437131685Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.437136105Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.437139965Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.446898058Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.446914728Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.537391508Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544232877Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544257117Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544269958Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544278088Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544284608Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544418572Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.544426962Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.576610424Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.583410801Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:37.623710558Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.759019126Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.766824532Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.766855553Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.766861124Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.766865974Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.766884844Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.776129152Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.776150213Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.864873652Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.872862563Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.872890674Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.872898164Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.872947116Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.872961596Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.873068809Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.87307684Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.906492637Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:37.913717716Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.077249181Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.084939324Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.084973635Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.084983215Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.084989045Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.084997246Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.094411738Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.094440839Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.185689441Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.193824937Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.193855578Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.193863998Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.193907039Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.19393685Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.194033883Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.194046614Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.227936745Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.235413601Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.398199416Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.405304941Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.405344133Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.405390584Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.405398124Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.405402454Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.415688872Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.415721523Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.511671962Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519619662Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519651113Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519658893Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519676293Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519689414Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519839038Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.519847698Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.554755349Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.562911865Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:38.629194675Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.737040188Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.745215605Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.745261956Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.745269796Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.745274977Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.745279347Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.754662378Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.754685889Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.842857742Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.849979509Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.85003499Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.850044411Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.850117603Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.850129503Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.850251297Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.850267477Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.885162537Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:38.893959412Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.076866738Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.08454267Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.084579922Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.084585682Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.084592332Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.084603782Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.094603592Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.094625513Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.189938483Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.197918104Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.197943334Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.197963285Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.197971075Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.197979406Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.19812438Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.19813419Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.232587018Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.240534828Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_890a5a3b-e222-4b29-81b1-a24f27413d31","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.424769703Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.432853267Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.432899929Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.432907999Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.432916489Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.432922119Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.442986141Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.443016331Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.542028189Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.55036915Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550413152Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550425522Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550470353Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550481934Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550653208Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.550663269Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.584493968Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.59249817Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:39.634651031Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.770165385Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778228479Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.77826104Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778324992Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778479146Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8036e7f6-3d88-4263-96f1-ac55d658c5c4 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57ff02a4-a47e-49dd-bdd2-e8b99a648043 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778542768Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778549738Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.77862082Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778826406Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:888"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778835006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778843207Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:914","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778847467Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:919"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778851137Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:924"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778865217Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778871347Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:930"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778899238Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.778906149Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.77896276Z","hostname":"sock","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"9e98d732-bacb-427d-8fc6-d17d91e3d4a0","remote_addr":"[fd00:1122:3344:101::1]:60559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3728645,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {07868725-02a3-4f16-aa72-fc5116f94a2b (dataset): DatasetConfig { id: 07868725-02a3-4f16-aa72-fc5116f94a2b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0b26c614-c5b0-495f-9921-fff24f62dbd9 (dataset): DatasetConfig { id: 0b26c614-c5b0-495f-9921-fff24f62dbd9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ec12956-4640-43d6-a39b-6e3a5dc39707 (dataset): DatasetConfig { id: 0ec12956-4640-43d6-a39b-6e3a5dc39707 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f029bfe-86b6-4b23-bdea-83512de4375f (dataset): DatasetConfig { id: 0f029bfe-86b6-4b23-bdea-83512de4375f (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0f1c66f5-054e-4f85-b96d-99bd4717c307 (dataset): DatasetConfig { id: 0f1c66f5-054e-4f85-b96d-99bd4717c307 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1dff6f76-8674-4277-b07c-072a61bea873 (dataset): DatasetConfig { id: 1dff6f76-8674-4277-b07c-072a61bea873 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 212c8858-971f-4450-8be6-797b7f642d45 (dataset): DatasetConfig { id: 212c8858-971f-4450-8be6-797b7f642d45 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 29f5fc67-abc8-4b81-b289-063fcfcbead7 (dataset): DatasetConfig { id: 29f5fc67-abc8-4b81-b289-063fcfcbead7 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2bdbd98b-ffee-44d7-9b52-3623ed2ef653 (dataset): DatasetConfig { id: 2bdbd98b-ffee-44d7-9b52-3623ed2ef653 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e094815-c5a6-4587-8dbf-67e1a7014ee3 (dataset): DatasetConfig { id: 2e094815-c5a6-4587-8dbf-67e1a7014ee3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e41ab10-a910-44aa-8c1c-cef50099e0a9 (dataset): DatasetConfig { id: 2e41ab10-a910-44aa-8c1c-cef50099e0a9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3318bb71-6e6f-40b5-a02f-6a0c5b5ce61b (dataset): DatasetConfig { id: 3318bb71-6e6f-40b5-a02f-6a0c5b5ce61b (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 38c5922a-5b54-4f54-88ac-72083a5bd076 (dataset): DatasetConfig { id: 38c5922a-5b54-4f54-88ac-72083a5bd076 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a85867d-efa4-4856-9c6b-52c04e084a4d (dataset): DatasetConfig { id: 3a85867d-efa4-4856-9c6b-52c04e084a4d (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3b7a5c25-940d-4bf4-99a5-083ebf6cdf50 (dataset): DatasetConfig { id: 3b7a5c25-940d-4bf4-99a5-083ebf6cdf50 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4196d4d5-b040-492b-921d-3c0fe3b2f186 (dataset): DatasetConfig { id: 4196d4d5-b040-492b-921d-3c0fe3b2f186 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 49af13f9-365c-496f-8629-545557f4ce6e (dataset): DatasetConfig { id: 49af13f9-365c-496f-8629-545557f4ce6e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 51f875d0-387c-43fb-ad9a-a033e5fb5a85 (dataset): DatasetConfig { id: 51f875d0-387c-43fb-ad9a-a033e5fb5a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5376ddc1-303e-4d36-829b-f6dc51f6c910 (dataset): DatasetConfig { id: 5376ddc1-303e-4d36-829b-f6dc51f6c910 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53ca730c-cb99-41c7-95c1-28d7f0d12801 (dataset): DatasetConfig { id: 53ca730c-cb99-41c7-95c1-28d7f0d12801 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5522f2bf-37d2-4dca-8178-8afe483841a6 (dataset): DatasetConfig { id: 5522f2bf-37d2-4dca-8178-8afe483841a6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5942032a-8e8b-4cbe-a6a0-ef250b75ff89 (dataset): DatasetConfig { id: 5942032a-8e8b-4cbe-a6a0-ef250b75ff89 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5d1f872d-2a0b-4d69-b723-24dc412c75e8 (dataset): DatasetConfig { id: 5d1f872d-2a0b-4d69-b723-24dc412c75e8 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 61807538-402d-4f12-8e1c-66e23e78c82a (dataset): DatasetConfig { id: 61807538-402d-4f12-8e1c-66e23e78c82a (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 65c4e466-ee1d-4a6f-837f-9bd1c545a960 (dataset): DatasetConfig { id: 65c4e466-ee1d-4a6f-837f-9bd1c545a960 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6678e69e-1be8-4822-9210-6abe5105e895 (dataset): DatasetConfig { id: 6678e69e-1be8-4822-9210-6abe5105e895 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 696edb80-13a5-4fa7-a781-9d34e9816834 (dataset): DatasetConfig { id: 696edb80-13a5-4fa7-a781-9d34e9816834 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b8e91bf-0a6d-4668-a053-4f0230322ade (dataset): DatasetConfig { id: 6b8e91bf-0a6d-4668-a053-4f0230322ade (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7543a616-2e0d-4151-ae49-79331ccf17ff (dataset): DatasetConfig { id: 7543a616-2e0d-4151-ae49-79331ccf17ff (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7901ff76-8ea7-46bb-8b28-52419d486cc7 (dataset): DatasetConfig { id: 7901ff76-8ea7-46bb-8b28-52419d486cc7 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f49789c-cac1-4f18-8257-2d2a84025d28 (dataset): DatasetConfig { id: 7f49789c-cac1-4f18-8257-2d2a84025d28 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f8a2cbc-7a4e-4fc3-9077-13fbbc6823d7 (dataset): DatasetConfig { id: 7f8a2cbc-7a4e-4fc3-9077-13fbbc6823d7 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 82eaf0ed-fc93-4c0b-973c-f60f621d4d5c (dataset): DatasetConfig { id: 82eaf0ed-fc93-4c0b-973c-f60f621d4d5c (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87311311-d5d8-4faf-92f8-03dce3cbd73a (dataset): DatasetConfig { id: 87311311-d5d8-4faf-92f8-03dce3cbd73a (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8916ca6e-3ea7-48ea-8fa0-1df4e7db03a3 (dataset): DatasetConfig { id: 8916ca6e-3ea7-48ea-8fa0-1df4e7db03a3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8bdd35c5-c694-4f69-bf3e-05bc7f7db0c8 (dataset): DatasetConfig { id: 8bdd35c5-c694-4f69-bf3e-05bc7f7db0c8 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8cb420dd-98bd-40f0-a192-3432a449022a (dataset): DatasetConfig { id: 8cb420dd-98bd-40f0-a192-3432a449022a (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8ef1c869-57e6-4dd6-bcf9-86eda9bb988b (dataset): DatasetConfig { id: 8ef1c869-57e6-4dd6-bcf9-86eda9bb988b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 927a3e25-354c-47d6-ac0a-f4c54c59199d (dataset): DatasetConfig { id: 927a3e25-354c-47d6-ac0a-f4c54c59199d (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 95f6ca7c-3175-4646-b73c-a56cc2a89561 (dataset): DatasetConfig { id: 95f6ca7c-3175-4646-b73c-a56cc2a89561 (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9688642a-eeb2-4f3b-bda1-8043da62ce4b (dataset): DatasetConfig { id: 9688642a-eeb2-4f3b-bda1-8043da62ce4b (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 990354e1-54a4-4687-9a28-c7e6bcbd5eee (dataset): DatasetConfig { id: 990354e1-54a4-4687-9a28-c7e6bcbd5eee (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c3e75e-5465-4a5c-b968-cdeb3289777b (dataset): DatasetConfig { id: 99c3e75e-5465-4a5c-b968-cdeb3289777b (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ed3b5fd-4273-4cca-96e6-54d6d8cb8399 (dataset): DatasetConfig { id: 9ed3b5fd-4273-4cca-96e6-54d6d8cb8399 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9f36a31f-42f0-46df-8818-c53c148060a5 (dataset): DatasetConfig { id: 9f36a31f-42f0-46df-8818-c53c148060a5 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a337e3b6-9c8b-47cd-9905-9570dd9b2a88 (dataset): DatasetConfig { id: a337e3b6-9c8b-47cd-9905-9570dd9b2a88 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a50ac93c-0fa9-457a-841b-7c75b794023d (dataset): DatasetConfig { id: a50ac93c-0fa9-457a-841b-7c75b794023d (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad4e63d3-c57d-481c-89a0-b538e0d6c48f (dataset): DatasetConfig { id: ad4e63d3-c57d-481c-89a0-b538e0d6c48f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b1490701-e44f-4f0a-8156-829d3f7a8d65 (dataset): DatasetConfig { id: b1490701-e44f-4f0a-8156-829d3f7a8d65 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b3522067-561b-49fc-9efb-adaff34f4fc4 (dataset): DatasetConfig { id: b3522067-561b-49fc-9efb-adaff34f4fc4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b5731d0d-1768-4b0b-9f53-322a8929d29b (dataset): DatasetConfig { id: b5731d0d-1768-4b0b-9f53-322a8929d29b (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b648ceab-9d0c-4d48-b57c-2b7c5793993d (dataset): DatasetConfig { id: b648ceab-9d0c-4d48-b57c-2b7c5793993d (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, bd5152b4-e566-4d58-8d2e-5eac1e5c84dc (dataset): DatasetConfig { id: bd5152b4-e566-4d58-8d2e-5eac1e5c84dc (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf5149e7-e3b3-486e-89f4-6e504f354a85 (dataset): DatasetConfig { id: bf5149e7-e3b3-486e-89f4-6e504f354a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8ed0881-3c0c-4651-bc36-6b175d3c54be (dataset): DatasetConfig { id: c8ed0881-3c0c-4651-bc36-6b175d3c54be (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ca6f5c64-90bc-4fc6-b2e1-f6e6728b065b (dataset): DatasetConfig { id: ca6f5c64-90bc-4fc6-b2e1-f6e6728b065b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d11873f3-0c17-405b-a55d-f221a6ae1b37 (dataset): DatasetConfig { id: d11873f3-0c17-405b-a55d-f221a6ae1b37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, df1e15e7-063d-4d5c-98eb-3e96ae6237da (dataset): DatasetConfig { id: df1e15e7-063d-4d5c-98eb-3e96ae6237da (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ea919201-f898-42f9-a77d-d62360bc7be9 (dataset): DatasetConfig { id: ea919201-f898-42f9-a77d-d62360bc7be9 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f0fbd416-591a-41c4-9ee7-d914def72328 (dataset): DatasetConfig { id: f0fbd416-591a-41c4-9ee7-d914def72328 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f34b78b2-0330-4ac9-9d11-8d79248ccb86 (dataset): DatasetConfig { id: f34b78b2-0330-4ac9-9d11-8d79248ccb86 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f9462eb4-b44d-49cd-9400-7b3bc9160eae (dataset): DatasetConfig { id: f9462eb4-b44d-49cd-9400-7b3bc9160eae (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb108cbe-787b-460b-8494-5d597c698476 (dataset): DatasetConfig { id: fb108cbe-787b-460b-8494-5d597c698476 (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fe5847ae-33a6-428f-aadb-37ff909728d1 (dataset): DatasetConfig { id: fe5847ae-33a6-428f-aadb-37ff909728d1 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fefebae3-bf48-4dc5-adac-90d51a97d2da (dataset): DatasetConfig { id: fefebae3-bf48-4dc5-adac-90d51a97d2da (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff19fce6-df53-452a-b45d-8d12f7b31462 (dataset): DatasetConfig { id: ff19fce6-df53-452a-b45d-8d12f7b31462 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.811382719Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:457"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.81140421Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33610"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.811806301Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:850"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {07868725-02a3-4f16-aa72-fc5116f94a2b (dataset): DatasetConfig { id: 07868725-02a3-4f16-aa72-fc5116f94a2b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0b26c614-c5b0-495f-9921-fff24f62dbd9 (dataset): DatasetConfig { id: 0b26c614-c5b0-495f-9921-fff24f62dbd9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ec12956-4640-43d6-a39b-6e3a5dc39707 (dataset): DatasetConfig { id: 0ec12956-4640-43d6-a39b-6e3a5dc39707 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f029bfe-86b6-4b23-bdea-83512de4375f (dataset): DatasetConfig { id: 0f029bfe-86b6-4b23-bdea-83512de4375f (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0f1c66f5-054e-4f85-b96d-99bd4717c307 (dataset): DatasetConfig { id: 0f1c66f5-054e-4f85-b96d-99bd4717c307 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1dff6f76-8674-4277-b07c-072a61bea873 (dataset): DatasetConfig { id: 1dff6f76-8674-4277-b07c-072a61bea873 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 212c8858-971f-4450-8be6-797b7f642d45 (dataset): DatasetConfig { id: 212c8858-971f-4450-8be6-797b7f642d45 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 29f5fc67-abc8-4b81-b289-063fcfcbead7 (dataset): DatasetConfig { id: 29f5fc67-abc8-4b81-b289-063fcfcbead7 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2bdbd98b-ffee-44d7-9b52-3623ed2ef653 (dataset): DatasetConfig { id: 2bdbd98b-ffee-44d7-9b52-3623ed2ef653 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e094815-c5a6-4587-8dbf-67e1a7014ee3 (dataset): DatasetConfig { id: 2e094815-c5a6-4587-8dbf-67e1a7014ee3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e41ab10-a910-44aa-8c1c-cef50099e0a9 (dataset): DatasetConfig { id: 2e41ab10-a910-44aa-8c1c-cef50099e0a9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3318bb71-6e6f-40b5-a02f-6a0c5b5ce61b (dataset): DatasetConfig { id: 3318bb71-6e6f-40b5-a02f-6a0c5b5ce61b (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 38c5922a-5b54-4f54-88ac-72083a5bd076 (dataset): DatasetConfig { id: 38c5922a-5b54-4f54-88ac-72083a5bd076 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a85867d-efa4-4856-9c6b-52c04e084a4d (dataset): DatasetConfig { id: 3a85867d-efa4-4856-9c6b-52c04e084a4d (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3b7a5c25-940d-4bf4-99a5-083ebf6cdf50 (dataset): DatasetConfig { id: 3b7a5c25-940d-4bf4-99a5-083ebf6cdf50 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4196d4d5-b040-492b-921d-3c0fe3b2f186 (dataset): DatasetConfig { id: 4196d4d5-b040-492b-921d-3c0fe3b2f186 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 49af13f9-365c-496f-8629-545557f4ce6e (dataset): DatasetConfig { id: 49af13f9-365c-496f-8629-545557f4ce6e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 51f875d0-387c-43fb-ad9a-a033e5fb5a85 (dataset): DatasetConfig { id: 51f875d0-387c-43fb-ad9a-a033e5fb5a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5376ddc1-303e-4d36-829b-f6dc51f6c910 (dataset): DatasetConfig { id: 5376ddc1-303e-4d36-829b-f6dc51f6c910 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53ca730c-cb99-41c7-95c1-28d7f0d12801 (dataset): DatasetConfig { id: 53ca730c-cb99-41c7-95c1-28d7f0d12801 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5522f2bf-37d2-4dca-8178-8afe483841a6 (dataset): DatasetConfig { id: 5522f2bf-37d2-4dca-8178-8afe483841a6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5942032a-8e8b-4cbe-a6a0-ef250b75ff89 (dataset): DatasetConfig { id: 5942032a-8e8b-4cbe-a6a0-ef250b75ff89 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5d1f872d-2a0b-4d69-b723-24dc412c75e8 (dataset): DatasetConfig { id: 5d1f872d-2a0b-4d69-b723-24dc412c75e8 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 61807538-402d-4f12-8e1c-66e23e78c82a (dataset): DatasetConfig { id: 61807538-402d-4f12-8e1c-66e23e78c82a (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 65c4e466-ee1d-4a6f-837f-9bd1c545a960 (dataset): DatasetConfig { id: 65c4e466-ee1d-4a6f-837f-9bd1c545a960 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6678e69e-1be8-4822-9210-6abe5105e895 (dataset): DatasetConfig { id: 6678e69e-1be8-4822-9210-6abe5105e895 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 696edb80-13a5-4fa7-a781-9d34e9816834 (dataset): DatasetConfig { id: 696edb80-13a5-4fa7-a781-9d34e9816834 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b8e91bf-0a6d-4668-a053-4f0230322ade (dataset): DatasetConfig { id: 6b8e91bf-0a6d-4668-a053-4f0230322ade (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7543a616-2e0d-4151-ae49-79331ccf17ff (dataset): DatasetConfig { id: 7543a616-2e0d-4151-ae49-79331ccf17ff (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7901ff76-8ea7-46bb-8b28-52419d486cc7 (dataset): DatasetConfig { id: 7901ff76-8ea7-46bb-8b28-52419d486cc7 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f49789c-cac1-4f18-8257-2d2a84025d28 (dataset): DatasetConfig { id: 7f49789c-cac1-4f18-8257-2d2a84025d28 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f8a2cbc-7a4e-4fc3-9077-13fbbc6823d7 (dataset): DatasetConfig { id: 7f8a2cbc-7a4e-4fc3-9077-13fbbc6823d7 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 82eaf0ed-fc93-4c0b-973c-f60f621d4d5c (dataset): DatasetConfig { id: 82eaf0ed-fc93-4c0b-973c-f60f621d4d5c (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87311311-d5d8-4faf-92f8-03dce3cbd73a (dataset): DatasetConfig { id: 87311311-d5d8-4faf-92f8-03dce3cbd73a (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8916ca6e-3ea7-48ea-8fa0-1df4e7db03a3 (dataset): DatasetConfig { id: 8916ca6e-3ea7-48ea-8fa0-1df4e7db03a3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8bdd35c5-c694-4f69-bf3e-05bc7f7db0c8 (dataset): DatasetConfig { id: 8bdd35c5-c694-4f69-bf3e-05bc7f7db0c8 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8cb420dd-98bd-40f0-a192-3432a449022a (dataset): DatasetConfig { id: 8cb420dd-98bd-40f0-a192-3432a449022a (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8ef1c869-57e6-4dd6-bcf9-86eda9bb988b (dataset): DatasetConfig { id: 8ef1c869-57e6-4dd6-bcf9-86eda9bb988b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 927a3e25-354c-47d6-ac0a-f4c54c59199d (dataset): DatasetConfig { id: 927a3e25-354c-47d6-ac0a-f4c54c59199d (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 95f6ca7c-3175-4646-b73c-a56cc2a89561 (dataset): DatasetConfig { id: 95f6ca7c-3175-4646-b73c-a56cc2a89561 (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9688642a-eeb2-4f3b-bda1-8043da62ce4b (dataset): DatasetConfig { id: 9688642a-eeb2-4f3b-bda1-8043da62ce4b (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 990354e1-54a4-4687-9a28-c7e6bcbd5eee (dataset): DatasetConfig { id: 990354e1-54a4-4687-9a28-c7e6bcbd5eee (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99c3e75e-5465-4a5c-b968-cdeb3289777b (dataset): DatasetConfig { id: 99c3e75e-5465-4a5c-b968-cdeb3289777b (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ed3b5fd-4273-4cca-96e6-54d6d8cb8399 (dataset): DatasetConfig { id: 9ed3b5fd-4273-4cca-96e6-54d6d8cb8399 (dataset), name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9f36a31f-42f0-46df-8818-c53c148060a5 (dataset): DatasetConfig { id: 9f36a31f-42f0-46df-8818-c53c148060a5 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a337e3b6-9c8b-47cd-9905-9570dd9b2a88 (dataset): DatasetConfig { id: a337e3b6-9c8b-47cd-9905-9570dd9b2a88 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a50ac93c-0fa9-457a-841b-7c75b794023d (dataset): DatasetConfig { id: a50ac93c-0fa9-457a-841b-7c75b794023d (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad4e63d3-c57d-481c-89a0-b538e0d6c48f (dataset): DatasetConfig { id: ad4e63d3-c57d-481c-89a0-b538e0d6c48f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b1490701-e44f-4f0a-8156-829d3f7a8d65 (dataset): DatasetConfig { id: b1490701-e44f-4f0a-8156-829d3f7a8d65 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b3522067-561b-49fc-9efb-adaff34f4fc4 (dataset): DatasetConfig { id: b3522067-561b-49fc-9efb-adaff34f4fc4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b5731d0d-1768-4b0b-9f53-322a8929d29b (dataset): DatasetConfig { id: b5731d0d-1768-4b0b-9f53-322a8929d29b (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b648ceab-9d0c-4d48-b57c-2b7c5793993d (dataset): DatasetConfig { id: b648ceab-9d0c-4d48-b57c-2b7c5793993d (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, bd5152b4-e566-4d58-8d2e-5eac1e5c84dc (dataset): DatasetConfig { id: bd5152b4-e566-4d58-8d2e-5eac1e5c84dc (dataset), name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf5149e7-e3b3-486e-89f4-6e504f354a85 (dataset): DatasetConfig { id: bf5149e7-e3b3-486e-89f4-6e504f354a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8ed0881-3c0c-4651-bc36-6b175d3c54be (dataset): DatasetConfig { id: c8ed0881-3c0c-4651-bc36-6b175d3c54be (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ca6f5c64-90bc-4fc6-b2e1-f6e6728b065b (dataset): DatasetConfig { id: ca6f5c64-90bc-4fc6-b2e1-f6e6728b065b (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d11873f3-0c17-405b-a55d-f221a6ae1b37 (dataset): DatasetConfig { id: d11873f3-0c17-405b-a55d-f221a6ae1b37 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, df1e15e7-063d-4d5c-98eb-3e96ae6237da (dataset): DatasetConfig { id: df1e15e7-063d-4d5c-98eb-3e96ae6237da (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ea919201-f898-42f9-a77d-d62360bc7be9 (dataset): DatasetConfig { id: ea919201-f898-42f9-a77d-d62360bc7be9 (dataset), name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f0fbd416-591a-41c4-9ee7-d914def72328 (dataset): DatasetConfig { id: f0fbd416-591a-41c4-9ee7-d914def72328 (dataset), name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f34b78b2-0330-4ac9-9d11-8d79248ccb86 (dataset): DatasetConfig { id: f34b78b2-0330-4ac9-9d11-8d79248ccb86 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f9462eb4-b44d-49cd-9400-7b3bc9160eae (dataset): DatasetConfig { id: f9462eb4-b44d-49cd-9400-7b3bc9160eae (dataset), name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb108cbe-787b-460b-8494-5d597c698476 (dataset): DatasetConfig { id: fb108cbe-787b-460b-8494-5d597c698476 (dataset), name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fe5847ae-33a6-428f-aadb-37ff909728d1 (dataset): DatasetConfig { id: fe5847ae-33a6-428f-aadb-37ff909728d1 (dataset), name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fefebae3-bf48-4dc5-adac-90d51a97d2da (dataset): DatasetConfig { id: fefebae3-bf48-4dc5-adac-90d51a97d2da (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff19fce6-df53-452a-b45d-8d12f7b31462 (dataset): DatasetConfig { id: ff19fce6-df53-452a-b45d-8d12f7b31462 (dataset), name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.811994207Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.812003537Z","hostname":"sock","pid":1572,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.812009127Z","hostname":"sock","pid":1572,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.812012937Z","hostname":"sock","pid":1572,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:905"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.812016277Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.812020297Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.91328019Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:39.913319101Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.004240273Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.004288434Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.09634701Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.096403232Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/debug\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.165940005Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.165980937Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.266259581Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.266305682Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.367102901Z","hostname":"sock","pid":1572,"name":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.367148472Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.456202101Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.456240832Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.54698956Z","hostname":"sock","pid":1572,"name":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.547039672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone\"), full_name: \"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.614935658Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.614971199Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:40.637383268Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.684971776Z","hostname":"sock","pid":1572,"name":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone/oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.685009847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone/oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\"), full_name: \"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone/oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.785016243Z","hostname":"sock","pid":1572,"name":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone/oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.785062365Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone/oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\"), full_name: \"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone/oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.883449514Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.883487585Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.987522707Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:40.987558228Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.0839751Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.084020361Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone/oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.183593685Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.183629706Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.282775527Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.282812318Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.368834909Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.36886647Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.457757584Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.457802346Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:41.484926621Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"16.029297634s"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.556415601Z","hostname":"sock","pid":1572,"name":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.556458302Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/debug\"), full_name: \"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.624096921Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.624132952Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:41.639190368Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.694082548Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.694135369Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.763133748Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.763169998Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.852162736Z","hostname":"sock","pid":1572,"name":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.852207307Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), full_name: \"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.950593396Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:41.950638837Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.049435878Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.049477129Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.139145666Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.139191467Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.224722874Z","hostname":"sock","pid":1572,"name":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.224765355Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.311516177Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.311564319Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.413077009Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.41311772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.510115779Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.51015887Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.596304175Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.596353296Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:42.641648858Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.694408256Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.694446866Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.794148624Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.794192575Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.893628245Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.893672946Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/zone/oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.992539999Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:42.992596311Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.097627192Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.097682314Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.196072403Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.196113414Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone/oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.296226153Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.296273875Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.393906872Z","hostname":"sock","pid":1572,"name":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.393962954Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/debug\"), full_name: \"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.460769828Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.460810059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.527659866Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.527699876Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.627491817Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.627540798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:43.644115448Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.716338889Z","hostname":"sock","pid":1572,"name":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.71638208Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\"), full_name: \"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/zone/oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.816663485Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.816715406Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.906118645Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.906154936Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.9957484Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:43.995792791Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.096995322Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.097039293Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/debug\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.166009021Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.166051232Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/zone/oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.266528471Z","hostname":"sock","pid":1572,"name":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.266573603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone\"), full_name: \"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.355787596Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.355832188Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/debug\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.427238205Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.427280147Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/debug\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.496523272Z","hostname":"sock","pid":1572,"name":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.496567783Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone\"), full_name: \"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.565398917Z","hostname":"sock","pid":1572,"name":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.565436298Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:44.646605618Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.65563991Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.65565835Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/zone/oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.756376197Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.756414348Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.845859948Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.845898709Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.93329248Z","hostname":"sock","pid":1572,"name":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:44.933326961Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.021978359Z","hostname":"sock","pid":1572,"name":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.02202515Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/debug\"), full_name: \"oxp_b55e819d-82d6-47a9-8489-82b572951fbc/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.097169166Z","hostname":"sock","pid":1572,"name":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.097218447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone\"), full_name: \"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.170345915Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.170386176Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/debug\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.236579723Z","hostname":"sock","pid":1572,"name":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.236620674Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone\"), full_name: \"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.302919454Z","hostname":"sock","pid":1572,"name":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.302958285Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/debug\"), full_name: \"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.368860634Z","hostname":"sock","pid":1572,"name":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.368895505Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.459288912Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.459323613Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:45.498143738Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4147276013193956983"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:45.498295452Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2130381667484272497"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.510956199Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4292"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.510973119Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4302","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.517892289Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4316","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.51791319Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4381"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.568504235Z","hostname":"sock","pid":1572,"name":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:949"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.568550096Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1331","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\"), full_name: \"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/zone/oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:45.649758678Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.676291187Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4447"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.676346058Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4474"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.683927238Z","hostname":"sock","pid":1572,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.684297539Z","hostname":"sock","pid":1572,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.684647418Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.684766532Z","hostname":"sock","pid":1572,"uri":"/datasets","method":"PUT","req_id":"5e72bd1c-dbad-4e33-aa58-6a9b3bc0f724","remote_addr":"[fd00:1122:3344:101::1]:33610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5873266,"response_code":"200"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.716923643Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4487"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.716958124Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4333"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.733016779Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.733248216Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54657"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.733543885Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.733549375Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.766321224Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.766351785Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.769465715Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4375"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.769522947Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4454","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.792064919Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4465"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.79210961Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4507"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.801350488Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.804090548Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.841028557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.841077939Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.841096169Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.841182562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.866226387Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4539"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.866259078Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.89463703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.894668451Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.894672331Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.956065138Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.961870627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.965988856Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.969077215Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4586"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.969156407Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Installing Omicron zone: oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.975977474Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.98239268Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:45.986626253Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.009641109Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:46.010190335Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.042285265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.042391848Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:731"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.04280348Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:734","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.04282217Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.079597265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143505696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143567678Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143579998Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143590158Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143598299Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143608589Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143618619Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.14364714Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.14365965Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143670301Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143680221Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143688161Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143696202Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143701862Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143707722Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143713452Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143718972Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143725422Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143731032Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143741363Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143747073Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143753123Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:46.143758853Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:46.623787494Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13863341701885100570"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:46.623901948Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14763723751087087726"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/c073d51c-abaa-48df-abb6-2f053e9c1bec)","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:46.651675072Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:47.619531399Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:47.619665653Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:47.619974092Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.103944296Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.104007208Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.585515112Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 51.863008ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.592613527Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.592655659Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.596358136Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:48.596418528Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 131.110075ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.653438749Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 71.730388ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.722012115Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 71.247044ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.728593405Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 143.981755ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.809439447Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 100.203909ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.911124711Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 128.305746ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.930540873Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 366.612152ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:48.976054251Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 202.634935ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.003699922Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 172.198545ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.083063541Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 481.684409ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.232044475Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 236.745943ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.265330929Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 411.048947ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.351468583Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 459.652396ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.510649813Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 472.717187ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.721568491Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 1.169686313s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.770258471Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 791.3029ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:49.980470388Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 888.229243ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:50.204923777Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 756.164994ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:50.782696389Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 640.244254ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:50.948762018Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 1.087990059s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:51.100576764Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 909.50052ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:51.5469428Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 1.39799907s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:51.597287648Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 823.331057ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:52.197112727Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 1.004535772s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:52.465651544Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9\") failed: Property not found. retry in 785.830746ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:53.004999992Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24\") failed: Property not found. retry in 947.866877ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:53.030397037Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817\") failed: Property not found. retry in 1.032276755s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:53.480362438Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.53265194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_2327cea1-dd20-406b-af0b-11f432f90f24"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.532690371Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_24da292b-72a6-4c9f-b124-daf5b3ed05d9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.532697071Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_internal_dns_b0f17ac9-6497-4365-ae94-118450acb817"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.532715812Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.533076782Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.533410282Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d9c18596-6a81-44e0-a8d1-3e6e39896724","remote_addr":"[fd00:1122:3344:101::1]:54657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8800035,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.535461141Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.588699253Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"failed to write DNS configuration (will retry in 58.860506ms)","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:54.618159256Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:675","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.695369352Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.74224686Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:691"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.74225847Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.773582157Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.774359419Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.77437451Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.803617897Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.80372638Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:52849"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.803887075Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.803892885Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.840208486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.840237657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.876567499Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.87659925Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.911150421Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.911190042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.911347317Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.911441449Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.922334055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.954678561Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.955451184Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.955464394Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.985746011Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 247, 78]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.985758591Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.986052089Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.98606347Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.986285386Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:54.986655407Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:55.011112505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:55.028652393Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:55.500297741Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4147276013193956983"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:55.500400224Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2130381667484272497"} {"msg":"Profile for oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:55.652629652Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:42:55.652820958Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 30.453442ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:55.780200447Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 116.188336ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:55.818919438Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 113.931812ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:55.946633006Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 522.703972ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:56.072362547Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 634.577826ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:56.604793836Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:56.626165895Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13863341701885100570"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:42:56.626293309Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14763723751087087726"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 632.903211ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:57.249163025Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 1.438254375s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:57.890784236Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 1.019173162s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:42:59.338848539Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9\") failed: Property not found. retry in 1.38918578s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:00.367580939Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.776764236Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.776804988Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.777148327Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.777492797Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a54e3c08-d765-4afa-a5c5-dd506bbe383c","remote_addr":"[fd00:1122:3344:101::1]:52849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6973710,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.780136184Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:732"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.810786912Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.810927206Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57882"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:01.820211494Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.820317628Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"84733ccc-324f-4e2c-8b41-7f5662afa71f","remote_addr":"[fd00:1122:3344:101::1]:57882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9230,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:01.82283681Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.953748682Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:01.953874285Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64997"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:03.817489132Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"909104461990024230"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:04.001738338Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13625217045218606091"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:04.535667969Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7565459126000326074"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:05.502115737Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4147276013193956983"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:05.50224163Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2130381667484272497"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:05.962990363Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:05.963087856Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"6601e820-5712-4adb-a099-72e6655ec58d","remote_addr":"[fd00:1122:3344:101::1]:64997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009153,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:05.965696872Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:06.08653564Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:06.086683244Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:56699"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:06.095326775Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:06.095454858Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"1e32ca70-a20e-4433-bbc2-c37cdafa7f6a","remote_addr":"[fd00:1122:3344:101::1]:56699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8700,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:06.098769594Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:06.251487717Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:06.25159534Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:43480"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:06.627755563Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13863341701885100570"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:06.627906308Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14763723751087087726"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a2be407a-d128-4fce-bd93-418f9e8fd8e9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-14T04:43:10.249444493Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.249538646Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"0bd8bc85-febd-4d1a-9dfb-199c74c8cc07","remote_addr":"[fd00:1122:3344:101::1]:43480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008890,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:10.252496972Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.079497682Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.079646227Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.088429971Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"278558bd-0647-488b-960e-b5ae958d11d9","remote_addr":"[fd00:1122:3344:101::1]:36458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8728,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.09184215Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:10.09184865Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.325893578Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.326009401Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.334081555Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"861847d6-085a-4ffc-ae59-d6429a222886","remote_addr":"[fd00:1122:3344:101::1]:59971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8026,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.337812213Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:10.337817483Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.671427753Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.671543566Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.680220668Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"afba44da-74a9-4e24-a79a-2fa1a33c4a2b","remote_addr":"[fd00:1122:3344:101::1]:36307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8622,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.682323429Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:10.682333819Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.872251139Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.872384942Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49514"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.880853398Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"c8b981ee-5131-44b0-bc0e-e32d0abcf66b","remote_addr":"[fd00:1122:3344:101::1]:49514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8412,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:10.882439134Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:10.882444964Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.183099363Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.183231676Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44833"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.203274327Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"83036a94-bc20-48f0-b944-fa1bdeee63c8","remote_addr":"[fd00:1122:3344:101::1]:44833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":19928,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 265050079, ip_addr: 15.204.87.223, stratum: 3, ref_time: 1731559391.079167, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.205953294Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.205974915Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1423"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.237125527Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.23723596Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42964"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.237443836Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.237449346Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.279097752Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.279122663Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.315993011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.316027502Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.354685012Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.354711132Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.38949864Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.38952061Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.428319584Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.428348615Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.465768108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.465794099Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.501323818Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.501351729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.539878955Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.539934496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.540229545Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.540351608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.572924051Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.572972532Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.572984293Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.572997043Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.573008703Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.573139597Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c34ed27b-b8c0-46aa-a620-955296135216","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":335817,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:11.573322592Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"125.91984ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c34ed27b-b8c0-46aa-a620-955296135216\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c34ed27b-b8c0-46aa-a620-955296135216\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.700582128Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.700957179Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.700978639Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.743362636Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.743405508Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.780623006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.780662407Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.813179508Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.81322091Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.85155611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.851601881Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.888372696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.888413587Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.925749388Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.92579481Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.960226817Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.960268498Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.999400241Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.999447812Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.999634868Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:11.999754981Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033396835Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033451057Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033471658Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033478948Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033487678Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.033620862Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"52aede19-6be1-4795-8e02-e90d19e7d853","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized)]","latency_us":332859,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.033824818Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"67.419711ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"52aede19-6be1-4795-8e02-e90d19e7d853\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"52aede19-6be1-4795-8e02-e90d19e7d853\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.102291491Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.1026328Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.102647441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.139712044Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.139757736Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.144676338Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"176.017225ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.17857814Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.178619071Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.214129889Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.21416478Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.250640767Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.250670598Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.288488473Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.288522694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.322478727Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.322518668Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.362603379Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.3626368Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.401124864Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.401174326Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.401390962Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.401519416Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.433695308Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.433740209Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.433748319Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.433754409Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.43377139Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.433899134Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"661cc7f9-396c-471c-b131-af3e893b73fc","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":331463,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.434093429Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"170.520554ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"661cc7f9-396c-471c-b131-af3e893b73fc\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"661cc7f9-396c-471c-b131-af3e893b73fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.6064724Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.606739458Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.606747378Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.646833439Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.64686229Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.685802988Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.685837559Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.725319612Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.725347573Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.766879995Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.766914927Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.803481875Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.803509746Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.839768796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.839798657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.877492589Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.87752633Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.915248542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.915283443Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.915469849Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.915598082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.947754184Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.947804865Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.947814985Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.947822575Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.947830276Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:12.94796988Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"918def36-3ea6-4d3f-86b7-868e4e48c7e3","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized)]","latency_us":341352,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:12.948184346Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"208.290041ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"918def36-3ea6-4d3f-86b7-868e4e48c7e3\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"918def36-3ea6-4d3f-86b7-868e4e48c7e3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.157958211Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.158282295Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.158296161Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.196275765Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.196312216Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.235057406Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.235090472Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.279906867Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.27994734Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.32040722Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.320443081Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.359492497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.359530542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.395349411Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.395383753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.436532112Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.436566959Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.475967551Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.476009871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.476298164Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.476420933Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.510257928Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.51028926Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.510297529Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.510304196Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.510380808Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.510514518Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"79cf336b-66aa-4a46-b483-3e0aa5227922","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":351398,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.510685438Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"116.193453ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"79cf336b-66aa-4a46-b483-3e0aa5227922\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"79cf336b-66aa-4a46-b483-3e0aa5227922\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.628303339Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.628713556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.628724395Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.671968645Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.672004046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.709044799Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.709084156Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.744514884Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.744549526Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.791268805Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.791314913Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.831056924Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.831091847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.866227173Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.866265424Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.904901217Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.904933083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.945427285Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.945462451Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.945738105Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.945849173Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.977661291Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.977696682Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.977703105Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.977709574Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.977717627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:13.977852396Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5697b056-01ab-4488-b721-35a79f4d55ef","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":348421,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:13.978008982Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"191.546754ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5697b056-01ab-4488-b721-35a79f4d55ef\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5697b056-01ab-4488-b721-35a79f4d55ef\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.171532093Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.171883572Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.17189006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.21483234Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.214873019Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.253434005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.253473963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.290250679Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.290285207Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.331162715Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.331199054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.372760011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.372791146Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.414227626Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.414262577Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.451617294Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.451654157Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.490799131Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.490846505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.491098231Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.491197083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.52452961Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.524557399Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.524561308Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.524567149Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.524585506Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.524683355Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e3846b32-cb51-44df-9207-af31ea998770","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":352046,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:14.524844001Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"202.71513ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3846b32-cb51-44df-9207-af31ea998770\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3846b32-cb51-44df-9207-af31ea998770\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.728812659Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.729165479Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.729182898Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.768271355Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.768304337Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.801732729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.801780852Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.841880336Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.841974763Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.875456573Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.875483349Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.910652665Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.910682581Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.944603831Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.944631507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.977488372Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:14.977515551Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.009786287Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.00982001Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.009994456Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.010086765Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.04064234Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.040678857Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.040687388Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.040693998Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.040700561Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.040819129Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b92d2930-11d6-4878-8b42-6a0b287c9f5f","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":311002,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.040969696Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"190.764148ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b92d2930-11d6-4878-8b42-6a0b287c9f5f\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b92d2930-11d6-4878-8b42-6a0b287c9f5f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.233305352Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.233597213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.233605632Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.270206054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.270228255Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.298950851Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.299238997Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ab5f8ea5-bd45-4c74-9fb7-5a2e52094085","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53344","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.304755289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.304771977Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.339339131Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.339383889Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.374703901Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.374743268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.407920714Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.407954024Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.442369809Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.442406186Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.479344382Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.479389009Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.512355036Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.512390691Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.512584055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.512687558Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.543670326Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.543691852Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.543696015Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.543700009Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.543703815Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:15.543865523Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"44aaaaa5-856f-4a2a-8547-822ffd88ed64","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized)]","latency_us":309565,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:15.544006452Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"537.213561ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"44aaaaa5-856f-4a2a-8547-822ffd88ed64\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"44aaaaa5-856f-4a2a-8547-822ffd88ed64\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.083922526Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.084323332Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.084333233Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.122462179Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.122485918Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.154698248Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.15471834Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.189916557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.189941325Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.222143217Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.222164254Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.258869102Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.258896272Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.291311441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.291332667Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.328534577Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.328561794Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.361601855Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.361641728Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.361841037Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.361957375Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.393934311Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.393976227Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.393983615Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.393991405Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.394000331Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.394207141Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f8457c87-081e-4bf1-8719-31bf8ff0a2fc","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized)]","latency_us":309241,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:16.394391947Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"344.215816ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8457c87-081e-4bf1-8719-31bf8ff0a2fc\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8457c87-081e-4bf1-8719-31bf8ff0a2fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.740724949Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.741115917Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.741129081Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.77995762Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.779990996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.81240006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.812441593Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.848070877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.848106391Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.880957911Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.880989937Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.917498224Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.917600351Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.950206578Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.95024481Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.989984724Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:16.990025581Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.023413537Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.023459935Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.023672763Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.023861973Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.057270393Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.057310408Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.057326898Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.057333245Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.057339039Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.057406786Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a9d768a2-b7a2-4a79-a6f1-b4535ba0b728","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized)]","latency_us":315663,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.05754013Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"476.482448ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9d768a2-b7a2-4a79-a6f1-b4535ba0b728\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9d768a2-b7a2-4a79-a6f1-b4535ba0b728\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.536056129Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.536407408Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.536476475Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.575394764Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.575425599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.609041626Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.609066461Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.645596793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.645621497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.679033757Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.679060767Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.716493704Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.716521061Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.751274616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.751300276Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.790057494Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.790084409Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.823195959Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.823230282Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.823424256Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.823527112Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.855666541Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.855715771Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.855724968Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.855732496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.855741308Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:17.855862868Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2f376000-2250-4d20-aebd-44cb954b6626","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":318796,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:17.856019417Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"598.944226ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f376000-2250-4d20-aebd-44cb954b6626\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2f376000-2250-4d20-aebd-44cb954b6626\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.458181498Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.458496789Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.458508151Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.498680559Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.498718603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.532508703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.532548089Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.570920646Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.570948791Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.604714238Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.604744436Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.643354874Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.643387359Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.685705379Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.685734377Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.723667121Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.723708119Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.759459192Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.759572975Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.759766255Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.759882509Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.792798471Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.792832908Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.792838917Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.792843708Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.792849529Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:18.792943779Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f9f29680-0280-4cdb-a8d9-8ee532bf5c5f","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized)]","latency_us":333715,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:18.793083817Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.200783712s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9f29680-0280-4cdb-a8d9-8ee532bf5c5f\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9f29680-0280-4cdb-a8d9-8ee532bf5c5f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:19.997835781Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:19.99807638Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:19.998084442Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.036265364Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.036293376Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.070696621Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.070724521Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.106734717Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.10676391Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.141452517Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.141484562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.178301422Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.178329998Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.212130164Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.212154324Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.247543795Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.24757073Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.282933323Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.282959536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.283142081Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.283245463Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315088979Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315124267Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315132058Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315137964Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315145475Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:20.315249343Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"124ef11e-9543-4359-a9a4-90cbb452bb4b","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized)]","latency_us":316498,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:20.315380006Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"866.045687ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"124ef11e-9543-4359-a9a4-90cbb452bb4b\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"124ef11e-9543-4359-a9a4-90cbb452bb4b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.185284199Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.185648982Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.185671051Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.21959279Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.219631641Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.257803433Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.25787718Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.291389251Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.291427081Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.32890731Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.328944296Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.363121152Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.363160688Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.39879145Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.398823729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.430988214Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.431028012Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.46773344Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.467774833Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.467965151Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.468078272Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.49938477Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.499412868Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.499434008Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.499439906Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.499444566Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:21.499535974Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"99d63202-370b-412d-a085-5a1860d31ba1","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":313277,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:21.4997568Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"709.170281ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99d63202-370b-412d-a085-5a1860d31ba1\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"99d63202-370b-412d-a085-5a1860d31ba1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.212086508Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.212442487Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.212461172Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.247880545Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.247920615Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.282979899Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.283013659Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.314206133Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.3142305Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.350234455Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.350262356Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.382639062Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.382668829Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.418200781Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.4182338Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.450669351Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.450699389Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.48757601Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.487607821Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.487777513Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.487881889Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.51937522Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.519494831Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.519505969Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.519513581Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.519522488Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:22.519644573Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"070d1e47-cab5-40b5-a230-e9d549429c3d","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":306527,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:22.519842935Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.71222894s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"070d1e47-cab5-40b5-a230-e9d549429c3d\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"070d1e47-cab5-40b5-a230-e9d549429c3d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.23778105Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.238148612Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.238158999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.27576447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.27578908Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.310178436Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.310272916Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.344018549Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.344045691Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.379457639Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.379484545Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.41588302Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.415903271Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.451460639Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.451480711Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.488472611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.488503849Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.523584978Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.523616779Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.523854544Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.523952291Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556179591Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556207736Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556215049Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556220964Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556226524Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:24.556327581Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"617a51ca-14f2-4124-9392-036901138d35","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":317513,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:24.556521555Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.05923705s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"617a51ca-14f2-4124-9392-036901138d35\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"617a51ca-14f2-4124-9392-036901138d35\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.620333595Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.620622729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.620634336Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.658436201Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.658467486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.693646527Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.693677738Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.729973098Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.730002433Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.765994286Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.766021671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.801310932Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.801343998Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.836899413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.836927483Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.873639603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.873666904Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.907256256Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.907288413Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.907474916Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.907639431Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937249603Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937302621Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937311649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937319439Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937325665Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:25.937411945Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f061a9fc-e091-4f91-b4a3-94aa5211e195","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":316106,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:25.937554909Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.164702865s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f061a9fc-e091-4f91-b4a3-94aa5211e195\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f061a9fc-e091-4f91-b4a3-94aa5211e195\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.107336415Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.107684162Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.107704329Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.145033917Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.145071034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.180648355Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.180682266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.216382973Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.216414314Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.251483762Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.251525097Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.287173108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.287286843Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.321526808Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.321558497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.356700077Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.356734804Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.391835064Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.391870821Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.392068706Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.392180009Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.415289104Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"468.291188ms"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.426199082Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.426235833Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.42624083Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.426248902Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.426255493Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:27.426368747Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"79e419df-b192-4610-a4b5-d644e7c56adc","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":318036,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:27.426505279Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.48396283s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"79e419df-b192-4610-a4b5-d644e7c56adc\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"79e419df-b192-4610-a4b5-d644e7c56adc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.915670177Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.91602685Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.916038241Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.9551053Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.955142847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.990032563Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:28.990076993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.025030893Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.025070841Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.058228233Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.058271546Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.093009626Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.093047558Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.127925208Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.127967499Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.163374775Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.163413954Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.198790787Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.198832225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.199021175Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.19913709Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.232112423Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.232139836Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.23214398Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.232148864Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.232160377Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:29.232270002Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"17824b03-b44f-4fba-afc2-1ed88c9c402c","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":315648,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:29.232401189Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.955333819s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17824b03-b44f-4fba-afc2-1ed88c9c402c\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"17824b03-b44f-4fba-afc2-1ed88c9c402c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.196565426Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.196832002Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.196841425Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.232372155Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.232399287Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.268396079Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.268431283Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.302509054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.302536477Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.338688374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.338721723Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.371325023Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.371353609Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.407303694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.407332552Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.44084963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.440878824Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.476612356Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.476645892Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.476813324Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.476914437Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510041963Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510078714Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510087115Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510094962Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510101318Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:32.510204024Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4aff8bd2-64d2-402c-a2f2-3033716924bf","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":312720,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:32.510413498Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"5.676822885s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4aff8bd2-64d2-402c-a2f2-3033716924bf\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4aff8bd2-64d2-402c-a2f2-3033716924bf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.204251913Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.20454504Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.204623515Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.242165469Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.242195592Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.275051224Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.275079397Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.309926409Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.309960291Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.344384863Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.344407749Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.38070611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.380736008Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.415923267Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.415947886Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.450507984Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.450528722Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.485722034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.485767334Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.485941931Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.486044271Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518092649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518122416Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518128697Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518135457Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518141851Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:38.518258807Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"304ff310-42e1-4ed2-abbb-97ecc9710482","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized)]","latency_us":313028,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:38.518493754Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"3.359398619s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"304ff310-42e1-4ed2-abbb-97ecc9710482\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"304ff310-42e1-4ed2-abbb-97ecc9710482\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.888093306Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.888410894Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.888423784Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.927198035Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.927231889Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.963832379Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.963877548Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.998604761Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:41.998637246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.035559754Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.035592361Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.070670854Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.070700817Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.107716915Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.10774612Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.142931498Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.142960478Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.178700853Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.178733676Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.178922719Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.179034115Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.210949403Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.210996195Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.211003948Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.211011776Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.211017364Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:42.211135389Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f895dfa5-09f5-4874-8331-ecf9f57d64d9","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized)]","latency_us":322030,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.211321188Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"5.347054382s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f895dfa5-09f5-4874-8331-ecf9f57d64d9\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f895dfa5-09f5-4874-8331-ecf9f57d64d9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:42.962665236Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"1.334756452s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:45.412256761Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:45.412388343Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"96225a8f-6835-4461-8e2a-ad0ef9cf31ff","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35000","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.574273024Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.574610628Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.574622703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.613498075Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.613529557Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.648612996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.648642069Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.683550458Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.683583075Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.718436633Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.718477059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.753652607Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.753685177Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.788228694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.788257973Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.822909356Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.822939516Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.857104747Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.857137973Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.857313694Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.857412997Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889565929Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889600495Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889607621Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889613696Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889620886Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:47.889778691Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"30a469d8-96e7-4dc1-88d1-e109d380b23f","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":314499,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:47.889937789Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.297416268s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30a469d8-96e7-4dc1-88d1-e109d380b23f\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30a469d8-96e7-4dc1-88d1-e109d380b23f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.200015148Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.200236583Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.200243586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.241643796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.241673441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.273007011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.273035727Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.308266772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.308313048Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.339800657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.33982643Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.374788121Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.374819659Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.406077038Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.406110489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.442712804Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.442738229Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.474147501Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.474185114Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.474367876Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.474472485Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507577671Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507603472Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507607606Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507627417Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507632957Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:43:52.507747363Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fdf0af64-18d3-4f1c-986d-05898b07687f","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":306781,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:52.507903237Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"8.083727964s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fdf0af64-18d3-4f1c-986d-05898b07687f\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:43:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fdf0af64-18d3-4f1c-986d-05898b07687f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:43:59.379830705Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"2.459249478s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.613806952Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.614148162Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.614156149Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.65231326Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.652366672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.686608465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.68664561Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.722111542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.722143126Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.757705489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.757741753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.79606769Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.796105528Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.828453345Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.828552835Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.864419988Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.864456617Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.900129111Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.900173934Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.900360952Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.900471861Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.931864278Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.931891409Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.931898554Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.931903373Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.93190897Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:00.931994004Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"31df5d76-b08c-4420-bc65-d2a89f425958","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":317161,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:00.932133695Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"11.936668728s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"31df5d76-b08c-4420-bc65-d2a89f425958\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:44:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"31df5d76-b08c-4420-bc65-d2a89f425958\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.901831977Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.9021767Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.90218599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.942559463Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.942585901Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.976706772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:12.976739304Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.014977336Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.015000333Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.04719928Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.047222755Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.083750453Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.083773039Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.118388132Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.118408654Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.152566166Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.152594873Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.185630683Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.185670453Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.185902657Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.185983482Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218178146Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218213716Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218222744Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218229466Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218237586Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:13.218356107Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8d1a7f0a-3118-4e37-b3cf-2ad2fcb8f014","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\", TimeNotSynchronized), (\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\", TimeNotSynchronized), (\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\", TimeNotSynchronized), (\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\", TimeNotSynchronized), (\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\", TimeNotSynchronized)]","latency_us":315514,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:13.218520859Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"16.384361504s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d1a7f0a-3118-4e37-b3cf-2ad2fcb8f014\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:44:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d1a7f0a-3118-4e37-b3cf-2ad2fcb8f014\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:15.526874741Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38319"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:15.52699001Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"35a17559-0c6f-406b-b91f-6757ecaaef78","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38319","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:16.922968429Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"4.059718129s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.647963306Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.648321719Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.648332472Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.687129901Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.68716219Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.724138005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.724185359Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.759777176Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.759804834Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.795257052Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.7952946Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.830493222Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.830519004Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.865834Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.865864779Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.902060644Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.902092126Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.938092943Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.93813229Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.93835515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:29.938456693Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.02353768Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.023581098Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.02358676Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.023591382Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.023594795Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.114795259Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.118677113Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.123071389Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.127553034Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.131931493Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.136084576Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.139948505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.145168368Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.150707158Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:30.155698549Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.088262083Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.088444231Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.133930487Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.134060119Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.139568244Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.139692344Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.149285394Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.149399453Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.155211329Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:33.155385015Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 31.684412ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.238667874Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 41.529493ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.275180735Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 65.829112ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.283156188Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 52.91326ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.290286501Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 43.212492ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.296739535Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 38.10663ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.315872139Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 79.808329ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.347590294Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 59.392266ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:33.555869138Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 115.027261ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:34.175705137Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 81.99595ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:34.32022445Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 230.111613ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.489091073Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 148.204764ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.638911814Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 171.332977ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.833016811Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 203.48006ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.86176301Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 268.624671ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.87707483Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 432.539028ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.889164603Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 401.870369ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.899852058Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 207.905217ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:35.921128186Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 514.675244ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.078475337Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.081552767Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"6.369394704s"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.053692102s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.140554137Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 290.532233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.155884835Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 509.073838ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.313839531Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 526.796691ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.336248297Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 668.540794ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.45949763Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 415.005332ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.607790672Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 661.376718ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.836053241Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 674.720268ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:36.874761015Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 655.937923ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.0355565Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.306881182s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.139471675Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 880.511237ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.208912858Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 1.24770543s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.509125823Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.285495931s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.561934844Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.044519304s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:37.70234508Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.296704553s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:38.101544497Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 624.828727ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:38.459773793Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 914.408308ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:38.760146712Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 803.992416ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:38.769749493Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.060051916s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:38.860023668Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.164589905s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:39.097097207Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.460725226s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:39.410971507Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 1.145256093s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:39.589293554Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 587.566106ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:39.690756849Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 562.922247ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:39.93608586Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.1571877s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:40.277237995Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.093404376s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:40.29063006Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.273048455s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:40.511405109Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 1.378609319s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:40.746791073Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 991.115654ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:40.884760725Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 813.457996ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:41.395929373Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 907.082691ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:41.446405774Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 568.094786ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:41.796668004Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 606.664565ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:41.887014474Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 974.351535ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.138692238Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.446384756s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.221034584Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 617.834158ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.365214983Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.308810809s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.375102005Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.11220424s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.509130792Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 615.927684ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:42.994447574Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179\") failed: Property not found. retry in 1.16214626s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:43.125892971Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.109625256s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:43.622266406Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 786.828736ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:43.633985268Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 581.061667ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:43.68060722Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 979.677569ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:43.695628138Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.209939455s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:44.274061729Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 662.243666ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:44.431709994Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 730.833751ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:44.685482383Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 961.056006ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:44.741544067Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 839.476408ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:45.103268041Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 721.914451ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:45.426649066Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.200332291s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:45.495959024Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:45.6574477Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:45.657677727Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bee79455-066a-4a95-b3da-80c6ca464aac","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37888","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.419548038s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:45.716090998Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.165613564s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:45.954732378Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.362137104s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:46.167520661Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 525.930407ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:46.707819427Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 748.415736ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.132600409Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.025716792s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.147651535Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 648.355233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.245417817Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 966.932738ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.539876795Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.486238825s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.889863371Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 850.940067ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:47.903395883Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 832.555786ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:48.183517345Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.429723796s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:48.516144519Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.166666878s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:48.762989568Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.256664888s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:49.026907281Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 797.726933ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:49.386312571Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 949.174739ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:49.938139383Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.315272977s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:49.953403295Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.427829683s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:50.193810159Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 938.598597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:50.29259487Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 644.476201ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:50.896700447Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 1.315821063s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:51.239868487Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.165657271s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:51.27724864Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.036961216s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:51.550932127Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 621.323696ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:51.630506101Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 683.599257ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:52.260856078Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f\") failed: Property not found. retry in 1.133981998s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:52.452162979Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf\") failed: Property not found. retry in 993.860621ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:52.564174573Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4\") failed: Property not found. retry in 1.367070924s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:52.59685828Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2\") failed: Property not found. retry in 1.020504672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:52.952415128Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005552125Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005602657Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_f35752ca-6f9e-4b3a-885d-07dd94a554f4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005609657Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_a5497784-bff1-4dde-82bb-a315dbf4b1cf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005615417Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_aa278e7d-acd9-4e87-9a7e-9aec3184740f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005622547Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_cockroachdb_858d5179-bfcb-433f-850b-f8251431f179"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.005674079Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.00606377Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.00641665Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"543a5ccc-6b73-4ff6-8820-900dd95b5294","remote_addr":"[fd00:1122:3344:101::1]:42964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24320680,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.042145865Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.070270359Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"074a74e0-f051-4c78-83bc-70d582c3d1ec","remote_addr":"[fd00:1122:3344:101::1]:46832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27963,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:54.070458285Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1152","retry_after":"134.681606ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"074a74e0-f051-4c78-83bc-70d582c3d1ec\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:44:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"074a74e0-f051-4c78-83bc-70d582c3d1ec\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.234762653Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"905d8f08-47b5-4c0f-a8a8-4457670d28ba","remote_addr":"[fd00:1122:3344:101::1]:46832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28987,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:54.234924267Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1152","retry_after":"168.409355ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"905d8f08-47b5-4c0f-a8a8-4457670d28ba\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:44:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"905d8f08-47b5-4c0f-a8a8-4457670d28ba\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.431672325Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"7a4a7d44-f78f-430a-b30e-8c894274476c","remote_addr":"[fd00:1122:3344:101::1]:46832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a3d061d-1f28-439c-bbca-7966622b18f2 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27540,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:44:54.431907792Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1152","retry_after":"147.978397ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a4a7d44-f78f-430a-b30e-8c894274476c\", \"content-length\": \"124\", \"date\": \"Thu, 14 Nov 2024 04:44:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a4a7d44-f78f-430a-b30e-8c894274476c\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:44:54.609658999Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3743"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:08.94233148Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.391953583Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3784"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.392446858Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"d2127ba0-cb40-41d4-9818-9cfe816759db","remote_addr":"[fd00:1122:3344:101::1]:46832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27813047,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.393065565Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.394283961Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"180ca849-5750-43c0-a12e-08d6f9f15f59","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49980","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":799,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:22.399046329Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"13.071741483s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b0f17ac9-6497-4365-ae94-118450acb817 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2327cea1-dd20-406b-af0b-11f432f90f24 (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3fb5c17e-f6d6-4b78-af51-c032b741af99, kind: Service { id: a2be407a-d128-4fce-bd93-418f9e8fd8e9 }, name: Name(\"ntp-a2be407a-d128-4fce-bd93-418f9e8fd8e9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: a5497784-bff1-4dde-82bb-a315dbf4b1cf (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: f35752ca-6f9e-4b3a-885d-07dd94a554f4 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: aa278e7d-acd9-4e87-9a7e-9aec3184740f (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: 858d5179-bfcb-433f-850b-f8251431f179 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4a3d061d-1f28-439c-bbca-7966622b18f2 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }, OmicronZoneConfig { id: 48221b8a-d832-4d05-ae47-e92f81388c0c (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 (service), filesystem_pool: Some(ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 79994e02-6747-4071-83af-7872b31640fb (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 459b411b-9021-43ae-8875-254e55794157 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: caaea8b3-b27c-497f-90e7-6c8a937e4888 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: cd56c423-2805-4083-b494-c6ede9d01104 (service), filesystem_pool: Some(ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d8f54c47-369e-495f-9807-788ae6a45735 (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6662bcef-9bb0-4db5-a8d9-0b955c8945fa (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a9726b39-d869-4fee-a30d-a3c774666257 (service), filesystem_pool: Some(ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: 793a53ed-62e5-4138-9ad1-3af0d0f10218 (service), filesystem_pool: Some(ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External } } } }, OmicronZoneConfig { id: 98d377e2-4d2e-4c64-ac1c-730a3b311abc (service), filesystem_pool: Some(ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External } } } }, OmicronZoneConfig { id: e352fa9f-1e84-413a-9abf-727b7916fdaf (service), filesystem_pool: Some(ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4c9f4209-1157-482f-88fe-3502ab2136c7 (service), filesystem_pool: Some(ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External } } } }, OmicronZoneConfig { id: 1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf (service), filesystem_pool: Some(ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External } } } }, OmicronZoneConfig { id: c2149298-6c2e-4280-ad72-81b85c939ab6 (service), filesystem_pool: Some(ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External } } } }, OmicronZoneConfig { id: c374e235-f9ee-49aa-a42d-1c691be9f25b (service), filesystem_pool: Some(ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External } } } }, OmicronZoneConfig { id: bbd9bba2-3ff6-446e-828a-28914b1070c0 (service), filesystem_pool: Some(ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.429666185Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.429935013Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64621"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.431074016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.431080796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 24da292b-72a6-4c9f-b124-daf5b3ed05d9, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.520840636Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.520884267Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: b0f17ac9-6497-4365-ae94-118450acb817, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.573878752Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.573923463Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 2327cea1-dd20-406b-af0b-11f432f90f24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.627261658Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.627309319Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a5497784-bff1-4dde-82bb-a315dbf4b1cf, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.66983468Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.669893262Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: f35752ca-6f9e-4b3a-885d-07dd94a554f4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.72437578Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.724419581Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: aa278e7d-acd9-4e87-9a7e-9aec3184740f, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.777624872Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.777671203Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 858d5179-bfcb-433f-850b-f8251431f179, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.833213781Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.833262403Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4a3d061d-1f28-439c-bbca-7966622b18f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48221b8a-d832-4d05-ae47-e92f81388c0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.888257855Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.888302106Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 48221b8a-d832-4d05-ae47-e92f81388c0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.943705691Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:22.943760523Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: caaea8b3-b27c-497f-90e7-6c8a937e4888, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.017584621Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.017646672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: caaea8b3-b27c-497f-90e7-6c8a937e4888, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9726b39-d869-4fee-a30d-a3c774666257, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.087022791Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.087069183Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: a9726b39-d869-4fee-a30d-a3c774666257, dataset_name: DatasetName { pool_name: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 793a53ed-62e5-4138-9ad1-3af0d0f10218, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.144694702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.144748363Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 793a53ed-62e5-4138-9ad1-3af0d0f10218, dataset_name: DatasetName { pool_name: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98d377e2-4d2e-4c64-ac1c-730a3b311abc, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.212079393Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.212140475Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 98d377e2-4d2e-4c64-ac1c-730a3b311abc, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e352fa9f-1e84-413a-9abf-727b7916fdaf, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.279562597Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.279610409Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: e352fa9f-1e84-413a-9abf-727b7916fdaf, dataset_name: DatasetName { pool_name: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c9f4209-1157-482f-88fe-3502ab2136c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.338542765Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.338590527Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 4c9f4209-1157-482f-88fe-3502ab2136c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf, dataset_name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.39359379Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.393644041Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: 1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf, dataset_name: DatasetName { pool_name: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c2149298-6c2e-4280-ad72-81b85c939ab6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.463789032Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.463844204Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: c2149298-6c2e-4280-ad72-81b85c939ab6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c374e235-f9ee-49aa-a42d-1c691be9f25b, dataset_name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.554832989Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.554895441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: c374e235-f9ee-49aa-a42d-1c691be9f25b, dataset_name: DatasetName { pool_name: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bbd9bba2-3ff6-446e-828a-28914b1070c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.638255625Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.638308697Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1379","request":"NewFilesystemRequest { dataset_id: bbd9bba2-3ff6-446e-828a-28914b1070c0, dataset_name: DatasetName { pool_name: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.705859063Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.705927425Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.706208883Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.706368068Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.807926068Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.823355915Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.823626792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.823661173Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.84737596Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.847604367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.848936465Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.875125524Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.87533133Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.875390952Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:23.876843974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076091124Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076159236Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076163996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076167846Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076171226Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076176366Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076181537Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076186747Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.076412733Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.081289785Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.086064453Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.091191791Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.096804024Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.102941721Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.255369116Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.259912537Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.264033537Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.268794555Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.273550882Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.278187196Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.283587483Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.288632469Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.29383981Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.299129193Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.304082926Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.309066841Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.315759685Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.321645165Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.378336557Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.425660127Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.472202925Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.472465653Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.480974009Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.488176758Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.495244582Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.502269306Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.509214307Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.515391166Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.522472721Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.584322172Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.643250979Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.645980218Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.646023459Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.699450396Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 240, 66]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.699491917Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.701786484Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.701807655Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.756184259Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 243, 126]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.75621692Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.758474476Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.758490476Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.812529631Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 230, 129]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.812563102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.814357604Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.814373844Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.870662974Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 150, 142]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.870723466Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.875537586Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.875554636Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931094275Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 137, 76]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931126045Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931339362Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931349132Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931503466Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931512897Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931593259Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931601829Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.93163335Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.931642611Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.933439912Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.934903585Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.934944926Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.936132221Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.938015985Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.938129628Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.938144339Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.94062132Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.940636501Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:24.940644871Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.100742688Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.106201796Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.114765254Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.115085163Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.123693232Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.132031024Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.186637395Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.205624745Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.243574074Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.254122929Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.273500731Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:25.347172044Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.71406329Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.722228286Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.726812019Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.727767217Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.72787401Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.735493831Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.828245946Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.828671918Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.82906685Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.837050761Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.83771391Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.838602016Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.838914085Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.838965037Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.847493724Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.847697409Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.847995018Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.848165973Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.854669591Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.86083878Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.866816083Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.873562618Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.880761427Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:35.888203412Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 68.981148ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.011800102Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 68.76172ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.027017192Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 49.320833ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.040245435Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 61.174483ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.054714485Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 46.626519ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.065853117Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 31.365479ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.084726904Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 144.239957ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.178775417Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 56.109513ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.482707479Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 42.647818ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.574341213Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 107.99634ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.826422472Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 114.609449ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.849707156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 64.259993ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.880164978Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 208.676052ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.942569956Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_459b411b-9021-43ae-8875-254e55794157:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:36.94446048Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 34.879901ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:36.986941771Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 69.772418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.132932509Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 62.516224ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.154828413Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 116.800298ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.184329977Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 64.798613ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.206710735Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 54.81784ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.226664323Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 56.854419ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.240612357Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 52.656113ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.258581187Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 130.476168ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.275008333Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 277.706297ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.292655604Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 122.627727ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.308871744Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 402.491321ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.325528016Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_459b411b-9021-43ae-8875-254e55794157 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:37.326050671Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 69.682985ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.35054184Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 192.957425ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.372573769Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 136.846877ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.388361376Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 147.711515ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.401891438Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 297.985838ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.571139879Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 149.636272ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.58601298Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 65.994602ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.607229904Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 196.968624ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.621402224Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 437.608515ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.636193413Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 523.442859ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.649507238Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 556.989223ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.663294048Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 235.136754ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.677511219Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 498.262131ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.691662829Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 256.506691ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.704680206Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 501.273291ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.717061875Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 196.622755ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.731941336Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 814.994946ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.745681443Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 35.019715ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.757758763Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 150.152013ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.771236774Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 134.048553ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.877191851Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 521.44817ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.891964599Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 243.640699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.906973454Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 441.594221ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.928674882Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 470.754779ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.944959324Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 278.254163ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.960629507Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 578.892507ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:37.979200415Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 217.050351ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.03252521Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 848.435439ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.092312391Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:38.209728411Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 652.846501ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.228356891Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:38.228547616Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.072941141s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.256012292Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 908.951385ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.271719297Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 318.319111ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.287966527Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.038009894s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.303921229Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 603.250052ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.32122671Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 992.406053ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.337541863Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 694.130183ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.388347954Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.023072384s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.430261758Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.112853697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.442189343Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 37.851177ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.5145925Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 149.845509ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.565066652Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 996.114366ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.577049379Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 756.186137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.592322971Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 585.183017ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.622437863Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 177.725925ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.817932724Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.229730463s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.89581743Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 640.686301ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.938262799Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.267038s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:38.953618384Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 360.435655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.010167651Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.30049865s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.095625636Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 852.188749ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.192772729Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 547.275394ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.220934625Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.22137585s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.34300524Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 574.358939ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.35475129Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.215882175s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.365734718Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.465928397s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.376949703Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 623.261757ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.388007403Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 616.892924ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.466214828Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 801.803777ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.566692718Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.439386947s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.584037571Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.003945939s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.596582694Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.467476239s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.783055504Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.305769756s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:39.942789859Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.240661151s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.026123272Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 793.23537ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.058369766Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.327313478s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.095473731Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 571.095148ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.138000272Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.450910123s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.233393735Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 519.495679ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.381939687Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 517.690522ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.407456536Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 726.504923ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.578944432Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.149285911s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.59476547Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.254701809s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.612956547Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.295810715s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.721071328Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 694.989937ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.855712326Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.104719886s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.868404484Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 735.109825ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.913740367Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.282083962s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:40.938172424Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 980.097535ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.03746198Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 645.60235ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.262563519Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.309689909s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.273800094Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.218967148s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.285340058Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 802.655683ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.317666654Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.358973572s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.436091354Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.325776804s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.564322907Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.030008712s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.65832602Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.302379521s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.695335472Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 508.364379ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.753172967Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 854.411327ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.880652117Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 632.278038ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.982663122Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 899.245082ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:41.994314549Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.176150739s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.029270971Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 508.968792ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.042534966Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.064204714s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.132879982Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.456549762s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.233678561Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.154044618s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.27507979Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.040749835s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.518364515Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.304299364s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.566168Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 914.059814ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.59622039Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.22269869s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.627030532Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 812.436ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.697113962Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 749.339507ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.743736032Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 956.359593ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.803327877Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.455738072s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.898535204Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.211585365s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:42.906817824Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.157163545s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.005655176Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.285668049s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.208601963Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 528.773923ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.218316355Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 594.806215ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.439089298Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 525.799381ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.503103132Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.056246434s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.517572011Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 645.93193ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.525373157Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 701.999488ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:43.569211817Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.037139078s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.102545161Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.228984987s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.291904045Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.131625261s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.45749231Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 630.042164ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.530421542Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 681.452135ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.591237763Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 789.53873ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.63914459Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 618.585693ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.688121919Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 615.71437ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.7523839Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 905.779878ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.816102464Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 757.51702ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.881162429Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.385333111s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.932058322Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.140958173s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.986959242Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 686.220207ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:44.995238302Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 724.908669ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.002831772Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.124949786s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.308435782Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.217317899s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.317420542Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.200200223s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.324874098Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 744.239898ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.331920852Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 725.634141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.377411849Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 704.937803ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.438060126Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 685.513791ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.539938976Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 909.902552ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.599594734Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.398322527s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.647667986Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.050304802s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.69062131Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.21371547s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.730993469Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.06743267s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:45.738794475Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 773.179534ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.086622188Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.301807598s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.155279406Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 905.184781ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.163739751Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 865.364998ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.171677421Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 969.19597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.234985394Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 869.645128ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.326523035Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.256326937s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.442897525Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 979.924355ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.518224177Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 618.290926ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.5345769Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 995.224062ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.543020715Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.000846117s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.749834364Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 995.93667ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.822142767Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:46.865307157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.105779399s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.882031622Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_79994e02-6747-4071-83af-7872b31640fb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:46.882287759Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:46.882353061Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_79994e02-6747-4071-83af-7872b31640fb zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:46.886487141Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 830.154853ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:46.954379267Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 43.82445ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.037063001Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 35.606673ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.044410384Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 968.77282ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.051950862Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 570.120403ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.059196942Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 698.546945ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.076440342Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 135.980244ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.088040498Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 107.669007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.094363701Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 994.457549ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.161437233Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 706.375914ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.349434497Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 113.426425ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.389821587Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.344969967s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.439385522Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 297.255837ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.458450434Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 524.458379ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.506066884Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 441.963999ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.519871993Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.063030641s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.532919111Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.196989773s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.552014264Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:47.554604369Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:47.554816685Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.493745975s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.643358429Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.124216279s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.712845612Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.004066431s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.763096497Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 293.820236ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.776848945Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.486164521s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.789351927Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 910.31301ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.801843888Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 44.727928ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.813584398Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 664.315132ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.830389395Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 93.513835ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.868595081Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 802.196881ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.973273993Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 241.875467ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.98148682Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 611.471064ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:47.998932946Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 662.10028ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.03049626Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 600.900366ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.040987744Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.024462067s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.067195743Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 999.747848ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.085837862Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 504.608484ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.192158941Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 266.805203ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.236931828Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 762.942916ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.507547335Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.10021186s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.518966236Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:48.586489991Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:48.586585984Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.290610276s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.60718304Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 899.631597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.620448504Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.334688056s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.65171708Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.461431938s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.704738156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.146244411s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.715281021Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 805.991255ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.726105904Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 643.10966ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.762301043Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 68.162599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.772307242Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 989.82396ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.782067935Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.071690216s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.792854037Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.19469274s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.803127254Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.246069224s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.858263351Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 124.812989ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:48.87446739Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 138.642514ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.016360279Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.220539258s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.099355913Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.239313747s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.111140944Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 983.360781ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.149211227Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 242.909299ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.166842687Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 744.504421ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.284437953Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 964.22936ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.297358697Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 576.076163ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.418036462Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 541.726157ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.428251737Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 685.073247ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.540526709Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 731.024912ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.557059598Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.304601542s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.635942222Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.071386873s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.784544986Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.222817021s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.876275052Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.259949034s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.890063711Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.258738633s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.909871655Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 713.948432ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:49.994172626Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 915.454188ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.012751904Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 614.697976ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.027317256Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 869.802336ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.041418365Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 714.071802ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.054535245Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.08583695s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.12104442Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 561.859183ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.149319609Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.426648009s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.17972488Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 854.105362ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.246087042Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.479569299s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.277920283Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.188534062s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.30646971Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 705.665277ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.336531131Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 618.317447ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.366011395Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.590101234Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"17.127573016s"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 866.714323ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.660200994Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 775.451952ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.721484509Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.480349567s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.733405764Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 959.085474ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.782576268Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.013179138s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.868618189Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.201402213s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.924532308Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 857.214735ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.94083267Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 831.259011ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.94979861Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 671.991141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:50.995872494Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.080225207s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.051499855Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 559.043683ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.109371671Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 687.89406ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.118631249Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.093209938s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.161592564Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 627.688611ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.178848873Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 794.985967ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.21673101Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 559.560296ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.506642876Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 801.325307ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.515971546Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.230920269s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.538473988Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.413489768s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.617458905Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 900.228753ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.679014948Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.091812457s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.689034758Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.146928162s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.750795267Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 994.596153ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.766704987Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:51.773667579Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:51.774057Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 541.630789ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.792729811Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 600.399434ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.808853297Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 721.799671ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.818569509Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.178374886s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.827325592Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.444733358s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.890052779Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 63.032569ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:51.988568452Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.27061685s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.024848262Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 128.493835ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.061318219Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 676.051931ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.079163065Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.485146292s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.195833644Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.496971995s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.261170826Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 138.96313ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.323239813Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 1.070142204s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.333516211Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 722.221804ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.344268273Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 946.793093ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.35245498Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.269443264s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.360163283Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 689.381464ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.424395763Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 287.594193ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.474253647Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 888.47153ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.551040801Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.404947542s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.589296809Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 616.462231ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.765401969Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 548.404862ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.774246074Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 850.006453ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.782652518Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 525.975143ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.793317067Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.299958285s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:52.803065188Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:52.803974305Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:45:52.990868327Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"34bc1119-7043-45cc-8e8a-e3116d3ba3f2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36823","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141704,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 700.739297ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.001778023Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.36167676s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.016749457Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.239009702s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.040200126Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 760.772921ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.076851367Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.463718035s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.123773506Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.459825807s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.304450718Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.342192022s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.312689947Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 532.382714ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.327040733Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 879.825995ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.334264942Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.434599094s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.343355845Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 984.426427ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.391642373Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 657.757907ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.417218474Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.355475162s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.448186821Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.02460131s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.638739539Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.055367639s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.647433651Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 817.995469ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.690639242Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.486882225s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.711365283Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 854.845021ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.768357803Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 530.772421ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.847314248Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 583.816848ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:53.868295296Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 630.256608ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.003990376Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 503.06625ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.08457703Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 544.220952ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.169136508Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 1.065343173s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.241096242Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.256629559s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.28866446Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.451352961s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.431820846Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.441421206s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.439723164Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.146794252s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.490395752Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 814.479615ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.525648583Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.075315446s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.555061815Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 1.336103625s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.600454349Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 843.434007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.612050825Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 619.047141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.636793322Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.237639171s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.64778855Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.212770047s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.663758623Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.051670361s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.687532671Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.159195045s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.712211956Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.309474906s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.722259347Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.113007661s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.772301566Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 916.435443ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.786035093Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 902.981627ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:54.812376795Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 727.675793ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.209219358Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 998.046239ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.266670212Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 1.323756039s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.315034142Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.152259232s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.350551181Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.302459278s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.468035233Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 924.684604ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.559706728Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 663.077922ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.644325468Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 930.613462ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.654686788Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 976.704668ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.713730638Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.130853552s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.728113845Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.170602593s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.751729489Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 701.538679ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.882256988Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 722.102063ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.892478704Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 596.396561ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.903642847Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.025916233s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.916531631Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 871.192127ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.930526766Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.457865175s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.941438152Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 1.118370426s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.95587924Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.292315904s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:55.96762478Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.078586507s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.042792737Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.400768588s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.280829051Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.482537102s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.319390647Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.496397339s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.500924734Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.201004852s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.514411165Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 847.543376ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.524987461Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 524.182817ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.601078735Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.338502631s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.615243295Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.093244821s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.626188092Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 652.812685ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.653188984Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 1.068066834s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.70656547Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 545.829532ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.791562301Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.236172289s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.815334729Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 649.649832ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.87236893Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.267027127s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.935762497Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 804.710639ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:56.956567299Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 856.929103ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.087313665Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.482097497s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.132272937Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.35119627s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.141842674Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 543.007007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.269877652Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 893.990167ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.315589916Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.23866364s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.346154551Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.214044883s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.383819532Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 922.392137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.408838926Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 1.462980177s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.535459684Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.273617172s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.691667617Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.196298725s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.724132487Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 596.888389ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.732428638Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.259553222s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.775042402Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 612.582362ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.785053812Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 930.499652ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.811842157Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.090722689s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.822260378Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 825.060655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.95386982Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 637.568355ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:57.962678625Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 549.383751ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.009024977Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 763.820004ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.061785465Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.254241789s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.212274793Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 773.324619ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.220544263Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.270813855s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.338772456Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 703.562569ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.346961724Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 713.94027ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.406091076Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.053807544s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.50256367Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.066616295s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.56680567Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 531.85613ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.594148722Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 839.716698ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.607934261Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 749.053186ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.616224702Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.251817074s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.624097099Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 625.870897ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.752416615Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 713.725448ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.789130849Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 684.216768ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.839712924Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.366516996s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.921810721Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 1.384011272s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.929275777Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 979.557685ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:58.973484117Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 1.364339457s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.002434036Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 831.77137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.009212222Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 963.89575ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.044271767Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 935.53189ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.059649403Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 827.580037ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.129319801Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 513.004678ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.136405546Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 899.472305ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.376267542Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 569.707408ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.38932104Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.021057022s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.45906517Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 1.35854726s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.475681501Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 1.261458284s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.513205088Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 562.583063ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.533281699Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.122659563s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.566570563Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 564.892282ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.618668452Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 692.603546ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.642892663Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 1.21578579s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.658032412Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 771.065913ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.851527795Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157\") failed: Property not found. retry in 1.423856078s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.885650103Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.222729303s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.962843448Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 871.418498ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.97222948Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735\") failed: Property not found. retry in 1.416060282s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:45:59.981376225Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 872.174925ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.006664367Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 1.373242226s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.017028077Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7\") failed: Property not found. retry in 1.400117752s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.10654807Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb\") failed: Property not found. retry in 1.226606971s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.19286542Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 597.521534ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.286390588Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa\") failed: Property not found. retry in 1.328942771s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.301547237Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 961.876856ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.323678928Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6\") failed: Property not found. retry in 1.230781694s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.34446874Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 580.203418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.37691797Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 760.302389ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.490132008Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 740.765615ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.631738649Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf\") failed: Property not found. retry in 1.362175936s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.698107561Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 768.830106ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.785148882Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc\") failed: Property not found. retry in 877.011075ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.844890341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0\") failed: Property not found. retry in 669.643961ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.853635064Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218\") failed: Property not found. retry in 633.212942ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.882576883Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257\") failed: Property not found. retry in 1.044759477s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.890185053Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b\") failed: Property not found. retry in 1.477924726s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.897880196Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 826.316955ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:00.965829684Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911\") failed: Property not found. retry in 1.42479632s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.195511305Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961\") failed: Property not found. retry in 1.267299785s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.261225688Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c\") failed: Property not found. retry in 772.462747ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.294807151Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104\") failed: Property not found. retry in 796.083476ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.380273726Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf\") failed: Property not found. retry in 721.390857ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.402325165Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694\") failed: Property not found. retry in 1.30633255s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.57896608Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 1.310969342s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:01.851494981Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888\") failed: Property not found. retry in 1.178668248s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:03.182931598Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419629201Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_nexus_13db289d-f9b6-41e9-b3bd-5ab4f3105694"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419686213Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_pantry_6662bcef-9bb0-4db5-a8d9-0b955c8945fa"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419694623Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_1a9e7c8c-1d00-45bb-a15a-3a0d395d7ebf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419700903Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_nexus_79994e02-6747-4071-83af-7872b31640fb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419711993Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_bbd9bba2-3ff6-446e-828a-28914b1070c0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419719274Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_external_dns_48221b8a-d832-4d05-ae47-e92f81388c0c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419725284Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_nexus_7d1b51fe-adf1-4350-9c25-9240054e3911"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419731214Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_clickhouse_caaea8b3-b27c-497f-90e7-6c8a937e4888"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419746424Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_pantry_cd56c423-2805-4083-b494-c6ede9d01104"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419753014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_external_dns_8c0d54f6-76b3-47cc-9828-b8e677cc2961"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419760855Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_oximeter_459b411b-9021-43ae-8875-254e55794157"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419766995Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_e352fa9f-1e84-413a-9abf-727b7916fdaf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419774385Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_c374e235-f9ee-49aa-a42d-1c691be9f25b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419779855Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_793a53ed-62e5-4138-9ad1-3af0d0f10218"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419785806Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_4c9f4209-1157-482f-88fe-3502ab2136c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419791496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_98d377e2-4d2e-4c64-ac1c-730a3b311abc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419798916Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_a9726b39-d869-4fee-a30d-a3c774666257"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419804766Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_c2149298-6c2e-4280-ad72-81b85c939ab6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.419810716Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3310","zone":"oxz_crucible_pantry_d8f54c47-369e-495f-9807-788ae6a45735"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.425296475Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.43202792Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.453127671Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ad68a274-93a2-44f7-90b5-2f61256a2eb3","remote_addr":"[fd00:1122:3344:101::1]:64621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42024049,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.498871846Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1446"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.499779532Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.500125802Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.781864311Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:787"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.796096123Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:799"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:04.853456904Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:06.613204164Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:06.753528188Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:06.882065859Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:06.989564502Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:07.085928533Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:07.267450119Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:07.39799898Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:46:07.758293354Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"71.156901874s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:07.835596662Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:08.346090465Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:08.719575631Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:09.306419375Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:09.975214652Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/f971eb12-0fea-4084-96d3-74198bddd289/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:10.734906812Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96030d57-c095-45df-b920-8840859d23be\", \"content-length\": \"173\", \"date\": \"Thu, 14 Nov 2024 04:46:11 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"f971eb12-0fea-4084-96d3-74198bddd289\\\"\", request_id: \"96030d57-c095-45df-b920-8840859d23be\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:11.954800337Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f13209ed-a61c-44f5-9641-6431ecc43f3b\", \"content-length\": \"173\", \"date\": \"Thu, 14 Nov 2024 04:46:13 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"f971eb12-0fea-4084-96d3-74198bddd289\\\"\", request_id: \"f13209ed-a61c-44f5-9641-6431ecc43f3b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:13.551376282Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:13.720049896Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8053157a-6876-46be-b4f7-7eb4e43adb3b\", \"content-length\": \"173\", \"date\": \"Thu, 14 Nov 2024 04:46:14 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"f971eb12-0fea-4084-96d3-74198bddd289\\\"\", request_id: \"8053157a-6876-46be-b4f7-7eb4e43adb3b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:14.898962135Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.763959334Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47374"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.766421966Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"14448933-60f7-4f98-8718-396676f704ec","remote_addr":"[fd00:1122:3344:101::b]:47374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2320,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.768810285Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46195"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.769799123Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.769830654Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.770416441Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.777192627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.783980754Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.784414897Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"064d8a2c-1680-436b-8dec-0ef08df241db","remote_addr":"[fd00:1122:3344:101::a]:44176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.795851648Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.797349341Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0630466f-3f58-437a-994a-857c40c917f3","remote_addr":"[fd00:1122:3344:101::c]:60773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.798821104Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.798841284Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.822570971Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41647"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.82288658Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.822932701Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.823055115Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.823653602Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.829018548Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56419"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.829871842Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.829939074Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.850048117Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.85015747Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.85017224Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.853717593Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.853845166Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.859574492Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.874179786Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65198"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.875661378Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.875680289Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.891550558Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.891661492Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b526a2f1-d6a7-4c74-9e43-db1387a8759a","remote_addr":"[fd00:1122:3344:101::b]:47374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":15665,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.892032532Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"47ae12fe-fe65-4191-9e85-fedd31f485fe","remote_addr":"[fd00:1122:3344:101::c]:65198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":17718,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.899938821Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.899961562Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.918669574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.918727636Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.927801328Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.928348164Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3caf4508-3e7b-4510-8f3f-26d6c0380d06","remote_addr":"[fd00:1122:3344:101::a]:44176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5768,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.932058171Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.93234727Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"00dcc441-7018-4174-932e-510f3e443948","remote_addr":"[fd00:1122:3344:101::b]:54984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.939279221Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.939302121Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"80be2a32-5c3b-4841-bbe5-fbb316f548cb","remote_addr":"[fd00:1122:3344:101::c]:60773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.942417671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.942466133Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.964257324Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.964342017Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.986468737Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:15.986545649Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.005440897Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.005509818Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.022506061Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.022582483Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.041283334Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.041347076Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.050845982Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.051086658Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3dbfdc1e-09fe-4386-ab7a-2bdf25cc1aff","remote_addr":"[fd00:1122:3344:101::a]:57138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.062343424Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.062370915Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.077411861Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7beca75d-6622-49ef-86d5-4b4385bb2d2e","remote_addr":"[fd00:1122:3344:101::c]:60773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":11399,"response_code":"204"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.077492513Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:469","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.077561205Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"770fc205-ede5-4645-9607-0d5cb6cfd354","remote_addr":"[fd00:1122:3344:101::b]:47374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3838,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.080924802Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.080945073Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.099591063Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.099659985Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.10743778Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:469","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.10777641Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"956e895b-ddfa-4857-b043-fbfa53f2f4d8","remote_addr":"[fd00:1122:3344:101::a]:44176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4870,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.118868211Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.118890002Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.138383286Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.138437288Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.156479321Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.156533792Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.174235315Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.174290736Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.186482989Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.186543701Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.200095733Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.200158595Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.213735679Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.216143748Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.228786614Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.228847376Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.232265445Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"9bd059ae-cb8a-47fc-8c0c-a21955763396","remote_addr":"[fd00:1122:3344:101::c]:46195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":463246,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.241174693Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.241197534Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.255678063Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.256019203Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.270376619Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.27042422Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.285852147Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"7ac98b35-3d85-4b92-bf39-5d2ece5aacba","remote_addr":"[fd00:1122:3344:101::a]:41647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":463053,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.290868352Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.290885213Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.302705885Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"5be200f0-99fd-49a4-9718-15c5d37c684e","remote_addr":"[fd00:1122:3344:101::b]:56419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":472925,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.995808215Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48605"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.99595613Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.99596536Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.996003841Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:16.99631072Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.008485693Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.008554365Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.035750512Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.035819374Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.061956661Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.062015263Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.086645186Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.086706128Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.112253608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.112318629Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.138684913Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.138739875Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.162383659Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.162441301Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.187583409Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.187665542Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.211170192Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.211236674Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:17.225852987Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"f28816b0-6a7b-4319-87fa-24af83f43189","remote_addr":"[fd00:1122:3344:101::c]:48605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229933,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.308825678Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48951"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310182637Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310427334Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310450115Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310480706Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310495106Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310509127Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310528097Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.310555958Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"077ca60c-1f33-49b5-9493-5a0536d38cd5","remote_addr":"[fd00:1122:3344:101::b]:48951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1433,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.312245167Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1083"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.732980351Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37746"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.733109225Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.733117975Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.733172537Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.733754783Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.745262156Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.745320788Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.76436981Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48890"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.764904745Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.764983498Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765034209Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765102111Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765138862Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765174763Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765241175Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.765261316Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bc6f2626-5737-400b-9427-ed8dd024efb9","remote_addr":"[fd00:1122:3344:101::c]:48890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":686,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.77057684Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.77059758Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.794487772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.794543974Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.818186418Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.81824163Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.841435782Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.841490113Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.864001215Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.864055167Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.887240988Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.887289729Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.909788507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.909842865Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.931558011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.934119856Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:18.944410704Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"cc6bb811-e6ed-43a9-88ac-bd6c181b1b3e","remote_addr":"[fd00:1122:3344:101::b]:37746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211359,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.79376304Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49585"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794019217Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794060528Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794088709Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.79411425Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794139381Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794169631Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794178762Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.794210313Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"183efde6-0962-4e47-9a30-b5b329057662","remote_addr":"[fd00:1122:3344:101::a]:49585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":357,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844265472Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51795"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844504828Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.84454832Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.84456017Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844582291Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844596791Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844614332Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844634522Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:19.844664323Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2386d2cd-bdb3-46d8-b625-b29fe218a142","remote_addr":"[fd00:1122:3344:101::b]:51795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":279,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:23.044484772Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:23.044594106Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"d9417384-7cf9-4378-bba9-4cb40956ec83","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35763","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:24.423533947Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:24.42362818Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2ed513e3-687f-4aa7-892d-9ff722c6940f","remote_addr":"[fd00:1122:3344:101::b]:65369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:24.440157088Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:24.440175859Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c3723845-40fb-4609-9f9b-3633cc48cbfc","remote_addr":"[fd00:1122:3344:101::b]:65369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:24.448639484Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9eab6e86-82ac-49a9-a163-1edca5a97324","remote_addr":"[fd00:1122:3344:101::b]:65369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.449994288Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.450124121Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5a27f40c-8e54-4b48-b749-ce7fa8947129","remote_addr":"[fd00:1122:3344:101::c]:48647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.525986658Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.526028179Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cd360cc0-ee60-4dbf-8bba-794b9b8f162e","remote_addr":"[fd00:1122:3344:101::c]:48647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.546255935Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a2215c7a-1163-4fd3-9108-0a87fd866498","remote_addr":"[fd00:1122:3344:101::c]:48647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":228,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.579317583Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43756"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.579418386Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"799ce63d-9c4a-42f6-9212-f5a31e25c3f9","remote_addr":"[fd00:1122:3344:101::a]:43756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.613454331Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.613579155Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"98cd8fdc-e219-4457-b9ed-1f40f8658458","remote_addr":"[fd00:1122:3344:101::b]:37710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.62272059Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.62273901Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d7cec7d2-73fb-47c9-b289-8d657b3b11a5","remote_addr":"[fd00:1122:3344:101::a]:43756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.632782851Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c3933fc7-78e5-4389-a806-31cc2f35501e","remote_addr":"[fd00:1122:3344:101::a]:43756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":213,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.645847729Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59538"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.646066806Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"36dbf99b-da6e-4d08-bab0-b74e14342131","remote_addr":"[fd00:1122:3344:101::b]:59538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.656288362Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.656303952Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"550a49a2-86b6-4dab-92b4-4e9d73f64404","remote_addr":"[fd00:1122:3344:101::b]:37710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.667634Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a968a23f-e2fe-4b21-becf-b20b1b1c4a97","remote_addr":"[fd00:1122:3344:101::b]:37710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.715489676Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.715661561Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9b2167eb-5f3c-49d8-bf51-3ff48a9787c1","remote_addr":"[fd00:1122:3344:101::a]:34690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.728540804Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.728678108Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"764c2a0e-ebfe-49a6-ba6c-244b8d72a3b0","remote_addr":"[fd00:1122:3344:101::c]:61846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.777357468Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43638"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.778695376Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.85585184Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.920931225Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"811e6be3-2d4d-4f5e-ab7e-589b53164c5c","remote_addr":"[fd00:1122:3344:101::c]:43638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143537,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.92110597Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40407"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.921144961Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44481"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.921338637Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.926750053Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"087a6702-7ed4-426b-b2b5-5a4f7d9bbc04","remote_addr":"[fd00:1122:3344:101::c]:43638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5345,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:45.968189023Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.028543271Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.028622424Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cff5d72f-d8a5-43e9-a619-f867ed394454","remote_addr":"[fd00:1122:3344:101::a]:44481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107373,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.032188437Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"90b1b68e-5457-48bf-b3bd-59361ffbb0f1","remote_addr":"[fd00:1122:3344:101::a]:44481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":351,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.063415241Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.071362891Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.071484495Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.072735211Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.073700089Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.073717329Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"7e5a974d-caf6-4a37-9de0-3fff134e6830","remote_addr":"[fd00:1122:3344:101::c]:43638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5505,"response_code":"204"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.081552916Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.081620318Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.081756562Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.081885576Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.081905696Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"e61f9469-b317-4af6-9922-a70a77635454","remote_addr":"[fd00:1122:3344:101::a]:44481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":573,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.120928246Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b7c50a44-75f9-46e0-98e0-fc0f0acf8833","remote_addr":"[fd00:1122:3344:101::b]:40407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":199554,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:46.121273527Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8a1793f3-6a7b-4b36-8484-2e922e2bdc28","remote_addr":"[fd00:1122:3344:101::b]:40407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:53.079114414Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54494"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:46:53.079695531Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"a8d41bcd-650c-4833-a8ef-d0f7879c268c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54494","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.684551101Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.684757977Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a13442ba-0cc3-44a4-84d2-cfbfbae8d217","remote_addr":"[fd00:1122:3344:101::c]:48660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.689447793Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.689576767Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d6af2ff2-c6f5-4cf4-a73a-f19a708c1db6","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.697055323Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.697181597Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6bcb3f43-e864-4f23-b132-5be3bfe6adc9","remote_addr":"[fd00:1122:3344:101::b]:59675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.707876446Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.707897327Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9cfff26b-1b2c-46b1-b490-9c85544c97c0","remote_addr":"[fd00:1122:3344:101::c]:48660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.718748241Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f8755c53-c19b-413e-8e93-307e5ab52011","remote_addr":"[fd00:1122:3344:101::c]:48660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.746468684Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.746491065Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b37605e9-805b-4aef-a9d4-bb1b0767caa0","remote_addr":"[fd00:1122:3344:101::b]:59675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.756747772Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"93293639-d5ac-4a30-8f0a-dcc8a485654b","remote_addr":"[fd00:1122:3344:101::b]:59675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.778253635Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.778274295Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d659cc0e-8def-4761-9087-c01d7f02e3e1","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.79052656Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1cb0ceba-1ab8-49c2-9d0c-728fa8567ef0","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":147,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.796327768Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46553"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.796476722Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"7ac8b172-7380-4a25-a04a-cecca390d474","remote_addr":"[fd00:1122:3344:101::c]:46553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.817322155Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61917"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.817464379Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e684e9cf-789f-4437-bea4-51818b5016f5","remote_addr":"[fd00:1122:3344:101::a]:61917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.880581207Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60626"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.88069438Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.918380961Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.974071404Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"351e92f9-c453-4fab-9994-2210482b3f83","remote_addr":"[fd00:1122:3344:101::c]:60626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93428,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.974292671Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58860"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.974350472Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.974408314Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0bf301af-f766-4596-8d9c-850446f2b22f","remote_addr":"[fd00:1122:3344:101::c]:60626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.974676352Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.979280945Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:15.979386628Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"35cfc44e-0798-4a3d-8ea2-ef33f4d24f05","remote_addr":"[fd00:1122:3344:101::b]:58860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4602,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.014893047Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.070434375Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.070521057Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7a04f946-0fb2-49da-8bc2-61cbe1d3b4d3","remote_addr":"[fd00:1122:3344:101::b]:43947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95959,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.074081481Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f10810b6-fc36-441c-8f08-5a8266c5f72f","remote_addr":"[fd00:1122:3344:101::b]:43947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.102684559Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.153767488Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"76fcaaea-e95a-48d0-9d52-cf22b7e9e572","remote_addr":"[fd00:1122:3344:101::a]:63984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174286,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:16.154143319Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0692a001-4acf-4cd7-ac68-1e90bd22b300","remote_addr":"[fd00:1122:3344:101::a]:63984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:23.119000819Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:23.119097462Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"dc90efce-9a55-4ad9-bd1c-1951e24eb00f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62875","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:25.44686981Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:42027","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:49652"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:25.519484633Z","hostname":"sock","pid":1572,"uri":"/c073d51c-abaa-48df-abb6-2f053e9c1bec","method":"GET","req_id":"ae4dc043-3955-4e7f-94d0-d050f0e9063e","remote_addr":"[fd00:1122:3344:101::d]:49652","local_addr":"[fd00:1122:3344:101::1]:42027","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71378,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:39.613666069Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.539218273Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.539398818Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1faf40e3-a94d-4dc7-94e9-166ce0109a78","remote_addr":"[fd00:1122:3344:101::c]:33604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.577031658Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.577184613Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"16bad91e-f59d-476e-9524-59fe701a0fbd","remote_addr":"[fd00:1122:3344:101::b]:41369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.626362007Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.62646761Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a42bcd40-0d2f-484a-9b5f-6922b5e4d2f4","remote_addr":"[fd00:1122:3344:101::a]:57746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.647076217Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59768"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.647296073Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.693713537Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.756919778Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"852702d7-1b9c-442c-9d14-1a7cceb8a5f2","remote_addr":"[fd00:1122:3344:101::b]:59768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109688,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757258667Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757281388Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757309269Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7437b7a6-7269-4d95-95a0-47a6bd83ad16","remote_addr":"[fd00:1122:3344:101::a]:57746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.75734783Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38308"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757401852Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46047"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757490024Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757503565Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54462"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757511315Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"64ba63dc-8bc1-4a6f-b554-b0f4975eb1a9","remote_addr":"[fd00:1122:3344:101::c]:33604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":283,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757538496Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43043"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757549776Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757559616Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"05105b5f-0f3d-4dfe-a118-1e3e0f9721d2","remote_addr":"[fd00:1122:3344:101::b]:41369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757735231Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8316080c-90b3-45ce-8e5d-c2bc254de7f8","remote_addr":"[fd00:1122:3344:101::a]:47380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757752612Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757827054Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"25b72449-b8b5-4da1-86f8-4c114839d8db","remote_addr":"[fd00:1122:3344:101::b]:38308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.757848335Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"47f8660d-d0f3-465f-b494-07697eacc0a8","remote_addr":"[fd00:1122:3344:101::c]:43043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.76355437Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a4a97431-3daa-4f8d-ad14-71e492dd5607","remote_addr":"[fd00:1122:3344:101::b]:59768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6130,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.769454281Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"35eae9f3-f993-4afc-bd41-9863bdf474dc","remote_addr":"[fd00:1122:3344:101::c]:33604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.769619296Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3ae760f9-c4c6-4903-90fe-843e117aca70","remote_addr":"[fd00:1122:3344:101::b]:41369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.769752989Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c661e85e-45a5-4b2c-9e0f-26017e6cc5df","remote_addr":"[fd00:1122:3344:101::a]:57746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.799442409Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.853870364Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.853955077Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"38336c47-5a18-4976-818a-b5efc2eb1b4f","remote_addr":"[fd00:1122:3344:101::c]:46047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96276,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.857599032Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"624897e6-c2a7-495b-ade1-b16c22d58cf9","remote_addr":"[fd00:1122:3344:101::c]:46047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":443,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.891395441Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.951261455Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2b4597ad-6c96-4bff-a2e9-4b324cbfad39","remote_addr":"[fd00:1122:3344:101::a]:54462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188173,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:45.951679407Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"16a64205-a420-43e9-a7e7-74bbde9802e7","remote_addr":"[fd00:1122:3344:101::a]:54462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:53.15246853Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:47:53.152583894Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2957d22a-6c81-4e6a-ab35-aac3bc435435","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41563","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.580653695Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.580755548Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0914ab6b-25e9-4bfc-9e93-0e093a5c9930","remote_addr":"[fd00:1122:3344:101::b]:52954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.670298221Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.670332782Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"02971a14-4680-4b00-a587-fa16ea107508","remote_addr":"[fd00:1122:3344:101::b]:52954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.680676772Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"91240550-d676-46e2-ac82-ba2eff371653","remote_addr":"[fd00:1122:3344:101::b]:52954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.721194565Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.721323438Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"eb17bd68-4308-4e71-9056-0cabefcc08ba","remote_addr":"[fd00:1122:3344:101::c]:48196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.742243464Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.742269165Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"edb2ffc4-b6a2-4331-b717-53c1aabd8af2","remote_addr":"[fd00:1122:3344:101::c]:48196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.753588023Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"68766018-82e2-4ba4-b728-cbff45d4e225","remote_addr":"[fd00:1122:3344:101::c]:48196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.828714988Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49662"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.82879583Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.872377782Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.931851065Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f150ea98-9440-4a2c-abad-dc5a3260cba4","remote_addr":"[fd00:1122:3344:101::c]:49662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103084,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932102552Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65493"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932192064Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52319"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932250686Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57317"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932296527Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38662"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932359769Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46902"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932495483Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"11a8ff1d-16e1-4b84-8211-933eaf0010cf","remote_addr":"[fd00:1122:3344:101::c]:49662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":321,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.932519284Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.936296843Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"282df09c-6d14-4e51-9ab2-5082d8ae4377","remote_addr":"[fd00:1122:3344:101::c]:46902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3754,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.942591216Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e3a213ce-5bd6-4460-8923-b7a7e9c2c860","remote_addr":"[fd00:1122:3344:101::a]:57317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6748,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.942708619Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0db3ed1f-225f-411b-8e35-18023a2d98ba","remote_addr":"[fd00:1122:3344:101::a]:38662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6732,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.94275073Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b423a7ce-3db0-49ad-bc11-bd2d5b3a23be","remote_addr":"[fd00:1122:3344:101::b]:52319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6657,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.960664469Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.96072219Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cfd81040-ae8e-480a-a547-7abf94f77216","remote_addr":"[fd00:1122:3344:101::a]:57317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.969966248Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5434428e-e8b1-4e45-a905-93fdee48f491","remote_addr":"[fd00:1122:3344:101::a]:57317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":452,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:15.977073224Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.032098297Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9dc238bd-5e11-4e75-933f-e9dd6eac7316","remote_addr":"[fd00:1122:3344:101::b]:65493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99751,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.032489769Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9acf4bdd-93e8-4dfd-bb0e-f5d046c71970","remote_addr":"[fd00:1122:3344:101::b]:65493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.096685118Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35113"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.096820192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.131711862Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.185080448Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"853e9aa8-64b3-4cbc-8075-a0826faeba7b","remote_addr":"[fd00:1122:3344:101::a]:35113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88317,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:16.185375306Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"76fe2a44-aaba-4ac1-8742-29f375c6d181","remote_addr":"[fd00:1122:3344:101::a]:35113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:23.188472275Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:23.188604609Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"19842403-f7ef-4627-b9b8-ec7c4b2e2bab","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59049","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.55610868Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63573"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.556237484Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"28453080-28c7-400c-bed8-4ade1282c9b4","remote_addr":"[fd00:1122:3344:101::c]:63573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.572278318Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.572298179Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4c84157-1db3-4023-9bcb-fbe1b2162284","remote_addr":"[fd00:1122:3344:101::c]:63573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.58926323Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.589434805Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"495389c7-73d4-42b1-bf64-ddd86f4ab122","remote_addr":"[fd00:1122:3344:101::c]:58160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.59928842Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bad45fd3-d0ee-439f-b06c-25d5600aa44a","remote_addr":"[fd00:1122:3344:101::c]:63573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.609738283Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.609856346Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"95a3f5b8-96c2-469c-800c-f8f79d5ea91f","remote_addr":"[fd00:1122:3344:101::b]:54983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.641229605Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38078"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.641327768Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.681626835Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741163189Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"84c7d44e-aa0f-43e9-b1f5-16a88bde8f6b","remote_addr":"[fd00:1122:3344:101::c]:38078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99875,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741435437Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50870"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741454097Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741502739Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"97d03ea5-9b54-498b-83df-97d2305cead9","remote_addr":"[fd00:1122:3344:101::b]:54983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.74153118Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741744166Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"80930bcd-80b6-4610-8964-e8446631671d","remote_addr":"[fd00:1122:3344:101::c]:38078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":291,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.741857239Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"813ec3b2-12a1-4998-98ab-8e2d8c1ebde7","remote_addr":"[fd00:1122:3344:101::a]:65312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.742292572Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8c98b817-2abe-4978-9607-64c54d100489","remote_addr":"[fd00:1122:3344:101::a]:50870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.743101065Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.74326555Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c7024304-105d-4fc7-bf28-a48892659f21","remote_addr":"[fd00:1122:3344:101::b]:64933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.754609988Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cd31e124-5270-4612-afc2-ac8b8910cfe1","remote_addr":"[fd00:1122:3344:101::b]:54983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.790947021Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.790962241Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1c5df756-8e59-40ea-82ed-e1a0fbcb07ef","remote_addr":"[fd00:1122:3344:101::a]:50870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.797654665Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5bd40333-1a08-4acf-9300-38d6920327c9","remote_addr":"[fd00:1122:3344:101::a]:50870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.867825306Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63662"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.867926439Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.907133055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.96981892Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9d5d7eb8-5673-4b30-b26c-568f5c6d4186","remote_addr":"[fd00:1122:3344:101::b]:63662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101926,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.970001905Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48264"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.970272343Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:45.975221137Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8eab82c7-9f87-461a-a6d8-ad37ece84cbb","remote_addr":"[fd00:1122:3344:101::b]:63662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4960,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:46.010174089Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:46.066761527Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"68719884-a538-4de6-a551-ed93d4bcc306","remote_addr":"[fd00:1122:3344:101::a]:48264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96616,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:46.067118968Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"041221ad-8a35-4f1b-ab02-3075a4fb1240","remote_addr":"[fd00:1122:3344:101::a]:48264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:53.221117196Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:48:53.221341333Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"37eadf75-64a3-4397-8284-f011929b434b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55231","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.61918361Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.619346305Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3ca01082-d9e5-478d-9600-1043ce3dd814","remote_addr":"[fd00:1122:3344:101::c]:55413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.68511614Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.685168231Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"80d6a59b-1230-4f53-b1a6-d5f230075fdb","remote_addr":"[fd00:1122:3344:101::c]:55413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.694663796Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6cf89c74-686f-4708-add0-9b98a528be0d","remote_addr":"[fd00:1122:3344:101::c]:55413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.754009225Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36092"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.754228721Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"816a5ce7-504a-46b3-bf03-75d71881bf85","remote_addr":"[fd00:1122:3344:101::b]:36092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.754259502Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.754335214Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ed61eb89-904d-405b-b1e3-ef4ff91f1bea","remote_addr":"[fd00:1122:3344:101::b]:51492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.765989012Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.766192338Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"05a8c2c2-13f9-4996-9c89-a8e465583161","remote_addr":"[fd00:1122:3344:101::c]:59748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.803508238Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.803525008Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"46c25bf8-a06c-4b39-a216-0cff2918e813","remote_addr":"[fd00:1122:3344:101::b]:51492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.812440326Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"11df84d1-b5e7-4437-b0e1-8adb1caffc45","remote_addr":"[fd00:1122:3344:101::b]:51492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":196,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.904652896Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57351"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.90476866Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:15.945796768Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.003816488Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"db27ed43-ca81-41a5-b04e-c91519619697","remote_addr":"[fd00:1122:3344:101::b]:57351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99090,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.003942212Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47227"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.003981613Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44531"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.004025534Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.004396625Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4d8354ee-7b23-4709-a972-37b9c5fe615d","remote_addr":"[fd00:1122:3344:101::a]:47227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.004430186Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0bccb9b4-1836-43f0-85f7-7e2a96d44c14","remote_addr":"[fd00:1122:3344:101::a]:56832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":301,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.004468807Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.004482547Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8e251b9f-6255-4b05-9788-a0643c4f4c7e","remote_addr":"[fd00:1122:3344:101::b]:57351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":269,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.045359531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.049355957Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.049436269Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f15baaf0-77c8-4fe7-ba50-96eee6f01e1e","remote_addr":"[fd00:1122:3344:101::a]:47227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":692,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.058233734Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5aaa9324-8b0c-4f01-98f4-ba7c8744f364","remote_addr":"[fd00:1122:3344:101::a]:47227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":264,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.105806292Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"60bcf7b3-ddcf-4c70-9234-a10b77df12ed","remote_addr":"[fd00:1122:3344:101::c]:44531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101462,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.106750509Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5a052086-0f25-4515-bbf6-02b22dfa4722","remote_addr":"[fd00:1122:3344:101::c]:44531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":691,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.259464941Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47466"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.259659167Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.294795345Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.348804358Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2dead8de-a43f-497a-b306-ee1d63192519","remote_addr":"[fd00:1122:3344:101::a]:47466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89214,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:16.34918728Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"faddd660-2dff-4fde-8192-1d6b26db0939","remote_addr":"[fd00:1122:3344:101::a]:47466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:23.25438871Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:23.254494173Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f3cd9f91-f4e4-4035-bf74-4ce1ed49b3ff","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53219","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.583014173Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54965"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.583129206Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"daaea1ae-321a-4d35-b9e3-24b1247f6393","remote_addr":"[fd00:1122:3344:101::c]:54965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.61191641Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50002"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.612006792Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c8578176-743a-4c21-9a93-5aba1d114f74","remote_addr":"[fd00:1122:3344:101::a]:50002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.633077802Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.633266688Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6822a75d-2fcf-48c5-b4eb-d20a9549b66f","remote_addr":"[fd00:1122:3344:101::c]:64346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.657537261Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.657568962Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8e6e0de8-c279-40c0-80fb-c8d34d794c6c","remote_addr":"[fd00:1122:3344:101::a]:50002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.666874051Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0695382c-0aaa-4831-a691-5e6944e30e8b","remote_addr":"[fd00:1122:3344:101::a]:50002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.672927187Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.673073601Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4353f003-c6f4-49e9-b2c0-37c4415cf8e3","remote_addr":"[fd00:1122:3344:101::b]:56787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.681264448Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34793"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.68135124Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.724796198Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.78286663Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d4337ba1-5657-4f73-820a-b0cd07be5d3e","remote_addr":"[fd00:1122:3344:101::a]:34793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101547,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783122038Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783156559Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3a67c5ee-3b78-4ec7-a8bd-195c56370b8c","remote_addr":"[fd00:1122:3344:101::b]:56787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783170069Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50668"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783261342Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51713"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783343374Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63077"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783355184Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783420906Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"28ad8b16-b2fe-4718-b396-24935ad11eaf","remote_addr":"[fd00:1122:3344:101::c]:54965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783523479Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2b9252de-b4ed-4bc3-bdda-d48b2bbdeddf","remote_addr":"[fd00:1122:3344:101::a]:34793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":306,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783571071Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d1299d26-7354-4b91-a23a-29ccf21ca3a2","remote_addr":"[fd00:1122:3344:101::a]:50668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783685044Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.783718945Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2c032076-7086-4e27-a3e8-4bca3e0b20ac","remote_addr":"[fd00:1122:3344:101::b]:63077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":246,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.793202439Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e909f68a-b805-409a-9415-954bdf0a91cf","remote_addr":"[fd00:1122:3344:101::c]:54965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":258,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.793373375Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f2eff722-ef83-4839-9bea-63045fce5fa6","remote_addr":"[fd00:1122:3344:101::b]:56787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.818528362Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.869031385Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"471b11ac-24ac-4b10-a7ee-3c50d317cd3c","remote_addr":"[fd00:1122:3344:101::c]:51713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85439,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.878334144Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"37ab9e7e-b22e-4b2e-a2fd-298b62a26f49","remote_addr":"[fd00:1122:3344:101::c]:51713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.939443894Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60167"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.939570377Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:45.978538016Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:46.036262968Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"09bc636c-2dbd-44d6-8c0f-a31e643d4478","remote_addr":"[fd00:1122:3344:101::b]:60167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96746,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:46.036666939Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c8ca05c0-0b2b-4ac1-94d2-679fae8c5d9d","remote_addr":"[fd00:1122:3344:101::b]:60167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:53.287711099Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:49:53.287897794Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0879f02f-badd-4371-b1cf-adbe52d14c08","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38264","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.677032358Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.677197793Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"97d639fd-1e08-4125-9911-369efe19ef2c","remote_addr":"[fd00:1122:3344:101::c]:50150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.752789462Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.752857934Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d02f21a5-3aa8-4b0f-b097-f90b97ad8ec9","remote_addr":"[fd00:1122:3344:101::c]:50150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.767857428Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"59172e0f-b7ce-47ab-a998-cadbbbb80d12","remote_addr":"[fd00:1122:3344:101::c]:50150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.799747021Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.799852164Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"05bec9fb-e58c-4a45-8580-c9b5b5384b27","remote_addr":"[fd00:1122:3344:101::b]:38496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.816427684Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61002"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.816547427Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"89f4bd63-d32b-465f-adc4-c0a8c9ac470c","remote_addr":"[fd00:1122:3344:101::a]:61002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.820552303Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.820570294Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3f51e5d0-d238-4df2-979f-424b4270f274","remote_addr":"[fd00:1122:3344:101::b]:38496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.831687946Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c2156455-8de2-4464-8043-55e57e57464f","remote_addr":"[fd00:1122:3344:101::b]:38496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":248,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.832232332Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.832243532Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0fa298dc-aa82-40f7-bc0c-6c62b571ed0e","remote_addr":"[fd00:1122:3344:101::a]:61002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.842327824Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bbc5606a-1207-4657-874f-db1070011c89","remote_addr":"[fd00:1122:3344:101::a]:61002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.887658047Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.887983166Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"909fcdad-3eff-426f-96de-afaa87c886cc","remote_addr":"[fd00:1122:3344:101::b]:46347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.895171824Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.89536314Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a8c606ae-73ed-4d2d-b876-cd6ab9f4b6c9","remote_addr":"[fd00:1122:3344:101::c]:64406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.937643174Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36232"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.937778968Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:15.986689614Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.070590304Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"639274f5-3fdf-4a4c-a860-3757feb5b8a2","remote_addr":"[fd00:1122:3344:101::c]:36232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132864,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.071011346Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b4b78e60-43f7-4aad-86bb-6379e4fe4450","remote_addr":"[fd00:1122:3344:101::c]:36232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.097687249Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.098012098Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e13f50d3-7a33-4b85-8e44-9848ae62bb1b","remote_addr":"[fd00:1122:3344:101::a]:35467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.127861803Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53611"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.128022337Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.173614018Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.225807339Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fee4af12-d9f6-4179-99dc-1dfd66190f9e","remote_addr":"[fd00:1122:3344:101::b]:53611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97874,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.225913042Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35909"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.226050636Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.258769314Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.31284665Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5d5c965a-0e9b-4f51-af05-29a2d00d282a","remote_addr":"[fd00:1122:3344:101::a]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86843,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.313113237Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"18424d71-3ce2-4ce0-a563-497143e513d8","remote_addr":"[fd00:1122:3344:101::b]:53611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:16.313333254Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a6da69ab-c0e0-44a2-8042-bc8d112c693a","remote_addr":"[fd00:1122:3344:101::a]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:23.321607533Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:23.321730626Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"cae9f0ec-6d5e-4194-b97c-a45f5b764401","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43199","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.532663221Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.532820096Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8b927c6e-a931-4019-97fb-f907e2ebee76","remote_addr":"[fd00:1122:3344:101::b]:41506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.539999744Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43505"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.540082816Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ca681ee6-bfc2-4127-b83e-b69700f82119","remote_addr":"[fd00:1122:3344:101::c]:43505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.547037768Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.547051788Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"978824f9-cacd-4177-a17b-7d0cccf554d1","remote_addr":"[fd00:1122:3344:101::b]:41506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.557367427Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f84ad85d-469d-4883-8c83-1ddb490d536c","remote_addr":"[fd00:1122:3344:101::b]:41506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.571225958Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.571413884Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fcfc0046-aa2d-4d13-a68f-41a214a1d8db","remote_addr":"[fd00:1122:3344:101::a]:65219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.583471793Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41515"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.58371924Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"47f7999f-60f0-4acb-a7a5-eda75fd51b1a","remote_addr":"[fd00:1122:3344:101::c]:41515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":147,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.587080857Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.587097428Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ce4ae5e7-83ec-4010-8267-fe5d258c586c","remote_addr":"[fd00:1122:3344:101::c]:43505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.596410218Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"def933ec-985d-4c37-96d2-4d46d56766ac","remote_addr":"[fd00:1122:3344:101::c]:43505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.627970392Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.627987082Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a140a02b-f2c6-4fd7-a1ad-4e87e0540598","remote_addr":"[fd00:1122:3344:101::a]:65219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.632836322Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60449"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.632929835Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.676097305Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.738865043Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c58d03d9-348c-44e9-95bd-9aa8b628a456","remote_addr":"[fd00:1122:3344:101::c]:60449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105982,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739004067Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33723"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739056978Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56151"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.7391121Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45678"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739154601Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50783"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739573793Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739590664Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1b610e32-2cc4-4329-8951-e879a735a052","remote_addr":"[fd00:1122:3344:101::a]:50783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":244,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739603984Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"22c4d1ae-70f3-4ad2-84ea-bc9111958acc","remote_addr":"[fd00:1122:3344:101::b]:33723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":334,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.739615815Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7e27a1da-07e3-405c-a927-a8408143d8bd","remote_addr":"[fd00:1122:3344:101::c]:60449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":434,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.743775005Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"88b5dc42-cfcf-470e-a0e5-84f04817c9f4","remote_addr":"[fd00:1122:3344:101::a]:65219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4557,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.77399194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.825571393Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.825661056Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"745c4a3e-f6b6-4300-ab31-0cd613db3902","remote_addr":"[fd00:1122:3344:101::a]:45678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86239,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.829148177Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1303a4eb-c543-4068-b2af-409102d103d1","remote_addr":"[fd00:1122:3344:101::a]:45678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.857964161Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.913442368Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d627193f-be5e-4f0a-a3bb-7e2097b9f773","remote_addr":"[fd00:1122:3344:101::b]:56151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174020,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:45.913797258Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5543d67b-b447-4e15-bc05-b0cfd03d7e05","remote_addr":"[fd00:1122:3344:101::b]:56151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:53.357776065Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:50:53.357890249Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bc25176e-983a-4c6d-b7f2-9cf30e7a8c00","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34272","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.656051579Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33696"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.656237245Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ae8cf38d-8434-4c55-a2a6-fc18e55aef8f","remote_addr":"[fd00:1122:3344:101::c]:33696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.717660093Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.717953872Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0e2a6da7-c95c-4f84-b595-e5ee90e8a392","remote_addr":"[fd00:1122:3344:101::b]:43246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.724510802Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.724530572Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5f0a9be3-ba7e-47d4-b33b-5868856f61e1","remote_addr":"[fd00:1122:3344:101::c]:33696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.735974374Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6450ebbe-b284-49bf-9e24-5bcdd5d7ccdf","remote_addr":"[fd00:1122:3344:101::c]:33696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.803861699Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.803958431Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f6716cf1-94c7-436b-9cfc-246f9f971e40","remote_addr":"[fd00:1122:3344:101::a]:48204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.823701933Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.823922729Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a2ab41d5-39a7-44f1-b064-345dc158ff0c","remote_addr":"[fd00:1122:3344:101::a]:50789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.866595505Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.866619936Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6a10908b-6b6a-4240-97b7-b17e6b191133","remote_addr":"[fd00:1122:3344:101::a]:48204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.876004528Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"079c89d8-2733-4e93-b181-503fc17d1006","remote_addr":"[fd00:1122:3344:101::a]:48204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.900448726Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.900562899Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ecd5c202-38c5-4b49-9879-520f00303080","remote_addr":"[fd00:1122:3344:101::b]:40999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.930528297Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35479"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.930766464Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"edc60807-f1c3-4fe1-aba8-8839bb186504","remote_addr":"[fd00:1122:3344:101::c]:35479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.945414658Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37374"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.945567732Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:15.99106054Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.061486019Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e01dabc1-6ea7-4354-a3ab-eb5a43428d9c","remote_addr":"[fd00:1122:3344:101::a]:37374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115995,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.06186559Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.061969293Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33267"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.062051725Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.062095247Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ffd5304d-dfe5-4b95-85ec-d8f24d8f7c93","remote_addr":"[fd00:1122:3344:101::b]:40999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.062164609Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"347f65c7-30f1-410b-bcc0-52989bfd123a","remote_addr":"[fd00:1122:3344:101::a]:37374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":301,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.06219489Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.075876726Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"929ce573-9b88-4b5d-999d-8e3b1fff3112","remote_addr":"[fd00:1122:3344:101::b]:40999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":283,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.107742329Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.150673782Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60252"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.150996481Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151055443Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151086634Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151122575Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151144956Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151161906Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151195387Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.151213248Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f6bfe9e8-e002-4aad-8f9a-ef3a56e27855","remote_addr":"[fd00:1122:3344:101::c]:60252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":421,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.165739238Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47653"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.166031727Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.166082338Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.166112749Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.1661316Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.16615007Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.16616374Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.166178521Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.166206522Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6b9eaa0b-de68-4f6a-9990-00ec1056b716","remote_addr":"[fd00:1122:3344:101::a]:47653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":360,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.180864556Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f72c8cf4-3680-46da-b293-71e6e2a945da","remote_addr":"[fd00:1122:3344:101::c]:42474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118743,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.180940448Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.186687445Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8b1c2943-b827-4441-aa4a-2b8842785194","remote_addr":"[fd00:1122:3344:101::c]:42474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":836,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.224114569Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.289669777Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"34995683-043a-4c88-92af-5e02b8f13773","remote_addr":"[fd00:1122:3344:101::b]:33267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":227498,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.290070539Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6d4bf619-cd93-4697-9ebb-2de825e6592e","remote_addr":"[fd00:1122:3344:101::b]:33267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.31325037Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33132"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313468416Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313514528Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313545489Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313567839Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.31359077Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.31360247Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313620771Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:16.313634601Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fc89927a-8bf9-4b0e-a3a3-770b256c86d7","remote_addr":"[fd00:1122:3344:101::b]:33132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":325,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:23.391311549Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53583"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:23.391886776Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"816925d0-a708-4291-9d3c-0f085913c620","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53583","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.577153398Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.577267852Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"38a0d3b1-d1a6-4f4d-affa-af30769ab391","remote_addr":"[fd00:1122:3344:101::c]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.624957543Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.624987464Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a6ece76d-479d-42a9-937d-79cc833ef250","remote_addr":"[fd00:1122:3344:101::c]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.632903943Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.633005806Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ba3cc4f7-8738-4fb3-81db-b0e04a53ae18","remote_addr":"[fd00:1122:3344:101::a]:53443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.633623844Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f544277e-d422-4eb4-a53c-01f80f1d2cba","remote_addr":"[fd00:1122:3344:101::c]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.634850779Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49967"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.634946252Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9a69fe25-ae3e-4b91-87da-793c5e050676","remote_addr":"[fd00:1122:3344:101::b]:49967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":18,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.675853217Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.675875467Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a54528c9-61df-4594-8303-e38cfdf67c7d","remote_addr":"[fd00:1122:3344:101::a]:53443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.680695587Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37132"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.680872982Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"33675d7f-95d1-4c6e-b63e-e2bb966d5ed4","remote_addr":"[fd00:1122:3344:101::a]:37132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.681385907Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.681407628Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c1dbfd45-7b1a-48b6-80e4-e8ca9a1f3053","remote_addr":"[fd00:1122:3344:101::b]:49967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.685707182Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c1dc457b-8bda-4121-9e1a-43e3d367f3f5","remote_addr":"[fd00:1122:3344:101::a]:53443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.689308566Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.689492822Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"861e698b-a1f3-4b7b-98bc-b99d6de8fa41","remote_addr":"[fd00:1122:3344:101::b]:58940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.694348602Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"504fda00-dc8f-411b-8079-67b85e33aa8f","remote_addr":"[fd00:1122:3344:101::b]:49967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.699527792Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50776"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.699598014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.741199999Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.798546719Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"00be316a-ddc6-4e56-973a-0f841f3e15ac","remote_addr":"[fd00:1122:3344:101::c]:50776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98978,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.798797026Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63507"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.798863498Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41076"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.79893473Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43922"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.799159997Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.799207048Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"45f1c766-df9c-4169-8280-48031ec0c9d6","remote_addr":"[fd00:1122:3344:101::c]:50776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":357,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.802794592Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"159ad54f-e712-4d72-8859-602839810a01","remote_addr":"[fd00:1122:3344:101::c]:43922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3697,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.834631664Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.882012196Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.882074008Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"be49032a-d96f-44cf-9f5d-52bc32398e83","remote_addr":"[fd00:1122:3344:101::b]:63507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83026,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.886151546Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"14ef0eb5-e424-4325-9956-4eaebb06f4d9","remote_addr":"[fd00:1122:3344:101::b]:63507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":735,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.918374929Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.970412406Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f2d8db09-1ae2-4b1d-90f7-2fb1e9c65cf9","remote_addr":"[fd00:1122:3344:101::a]:41076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":171284,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:45.970786047Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7a05d11e-2855-42af-829c-a3d051de22fb","remote_addr":"[fd00:1122:3344:101::a]:41076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:53.424745822Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:51:53.424892796Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"6cc095e5-6c5c-497f-b80b-125598e9bad3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59187","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.721397227Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.721577313Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"81b26ce1-7941-4500-a305-99fad9df4a18","remote_addr":"[fd00:1122:3344:101::b]:41742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.721627374Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.723007294Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9dddce36-a781-48e1-8ee6-0d1f37a89018","remote_addr":"[fd00:1122:3344:101::c]:47924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.766294188Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.766316198Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0aaaaea4-70fa-4e66-bb34-b54f892ff17d","remote_addr":"[fd00:1122:3344:101::c]:47924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.767690348Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.767708029Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7561b011-4219-4120-b26d-417b93ab9ef1","remote_addr":"[fd00:1122:3344:101::b]:41742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.776682428Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ac4feeec-d700-49be-ae59-37715d857677","remote_addr":"[fd00:1122:3344:101::b]:41742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":223,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.810768805Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57613"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.810991291Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"ca444394-7651-467e-adc3-13f305f7c3c1","remote_addr":"[fd00:1122:3344:101::c]:57613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.817290084Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9913e934-d8f1-40e2-9407-334f6b05204c","remote_addr":"[fd00:1122:3344:101::c]:47924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.894337605Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.894421527Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ead6f1e4-8eda-4ba5-a58c-60d601398f12","remote_addr":"[fd00:1122:3344:101::a]:57058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.898951419Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35400"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.899034421Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:15.936738093Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.001335944Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"15f9328e-9c30-4b72-b6e0-4d8d648e9ec3","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102323,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.001474937Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45821"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.001511498Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51570"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.001783486Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.001841378Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.006584616Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8966d33a-8c4b-4789-b72e-57322b652331","remote_addr":"[fd00:1122:3344:101::b]:45821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4755,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.00672899Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cf884ebd-e774-4f94-82b7-5271f36683a3","remote_addr":"[fd00:1122:3344:101::c]:35400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4928,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.00675115Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5382dc42-b2c9-4a31-8df2-ba1cc2c18afb","remote_addr":"[fd00:1122:3344:101::a]:57058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4943,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.042180036Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.080512736Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35515"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.080896088Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e1382954-4587-46fb-ab5b-032a53794e2a","remote_addr":"[fd00:1122:3344:101::a]:35515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":255,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.092671329Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5affec41-c98b-439d-82b3-9fefaee72c5b","remote_addr":"[fd00:1122:3344:101::a]:57058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":804,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.100965509Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42588"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.103828872Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.103839932Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"825aae5c-1419-425f-bbd4-9aab5d4aed6e","remote_addr":"[fd00:1122:3344:101::b]:51570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102176,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.108920949Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d0eefd96-71cf-437c-8843-e991ff3f610c","remote_addr":"[fd00:1122:3344:101::b]:51570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":778,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.145792247Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.204298141Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"59bcc303-4760-48e1-afee-d02af3a14a65","remote_addr":"[fd00:1122:3344:101::a]:42588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103259,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:16.204585009Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e0f15289-5b9c-4918-b794-820078ddecd3","remote_addr":"[fd00:1122:3344:101::a]:42588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:23.461053014Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:23.461146697Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e02f4f23-28e8-422c-b778-25995575947c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62491","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:25.257028133Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:25.258201397Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:25.293062257Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999584s","next_cleanup":"Instant { tv_sec: 1924, tv_nsec: 61667502 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:39.613179694Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.561159928Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46851"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.561326602Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2fd88009-0216-4610-83c1-16160c0dfaa7","remote_addr":"[fd00:1122:3344:101::c]:46851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.572062793Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49882"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.572211908Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"94c32390-d739-4348-a558-4c669d7b7822","remote_addr":"[fd00:1122:3344:101::b]:49882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.572404663Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.572527917Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"dc936cfe-765a-4ac4-ad08-796b291d886a","remote_addr":"[fd00:1122:3344:101::a]:43719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.614633806Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.614674838Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4440f158-4c7f-48cd-bde9-20c5d794a963","remote_addr":"[fd00:1122:3344:101::c]:46851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.624515723Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0d577dc9-205a-4dcc-b749-8dffe38eda2c","remote_addr":"[fd00:1122:3344:101::c]:46851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.626976554Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.626992874Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"769ea59f-ab72-4854-8877-09e4d271b3ae","remote_addr":"[fd00:1122:3344:101::b]:49882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.62855478Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.62856936Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b3c7a2c0-058b-4df9-a4fc-8f16f20b8c1b","remote_addr":"[fd00:1122:3344:101::a]:43719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.63927282Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3bd4ea68-9644-43d0-bf66-4c3a1c488354","remote_addr":"[fd00:1122:3344:101::b]:49882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.639891028Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42031"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.64169262Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.682125131Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.743658703Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2080517e-ba5e-47e4-b1c9-74d189d3fb59","remote_addr":"[fd00:1122:3344:101::b]:42031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103206,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.743931831Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42938"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.743992303Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56339"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744045134Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45189"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744106466Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744196208Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744376023Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c61d291f-2bd5-4c27-8e9f-e7cecc681b50","remote_addr":"[fd00:1122:3344:101::b]:42031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":355,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744431485Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9d11a461-622b-435a-8e85-19389b66859f","remote_addr":"[fd00:1122:3344:101::c]:42938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":346,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744516928Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744549908Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c90f8a76-853d-47e0-a0ed-fd750fb2c159","remote_addr":"[fd00:1122:3344:101::a]:43719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":747,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.744665722Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"de39fa44-b252-4b7b-804c-a992ff1717eb","remote_addr":"[fd00:1122:3344:101::b]:49239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":311,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.750382588Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"64e044ff-80e5-4879-9834-7356d6d90911","remote_addr":"[fd00:1122:3344:101::a]:42299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5802,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.783217658Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.840030853Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.840091485Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1041aa2e-e221-4bfc-b6c4-567c632abd70","remote_addr":"[fd00:1122:3344:101::a]:56339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95725,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.844016288Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"312ae5db-b38e-4a19-aaaf-06fb81d90ea7","remote_addr":"[fd00:1122:3344:101::a]:56339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":711,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.875740397Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.932970474Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e3a1d7aa-ff7b-4c16-b21d-1b29f0e91410","remote_addr":"[fd00:1122:3344:101::c]:45189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188600,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:45.933331955Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"64b75bcf-baea-496d-9f2b-34ed59bb49e2","remote_addr":"[fd00:1122:3344:101::c]:45189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:53.494737881Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64764"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:52:53.494922476Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5b8e4c50-0e77-47a5-b7af-12bc2c16bb8a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64764","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.578020581Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51045"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.578789173Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"253c77ca-eddb-488e-af38-d057a5ba82a8","remote_addr":"[fd00:1122:3344:101::c]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.663186277Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.663219948Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"deff4467-467c-4fb5-ae72-8289cf7b5b97","remote_addr":"[fd00:1122:3344:101::c]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.677044628Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4a999afa-9040-4993-ae6e-34b806e4d92c","remote_addr":"[fd00:1122:3344:101::c]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":231,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.735556563Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.735660086Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9ff0c5c7-9ba0-4b7d-b1e2-7ad24cad2f8d","remote_addr":"[fd00:1122:3344:101::b]:61433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.780533005Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.780565076Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2db00bbb-555d-4c27-bb9f-b6d9c9a38c8b","remote_addr":"[fd00:1122:3344:101::b]:61433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.790300558Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1f097234-6faf-42d0-902a-c3238e1d0046","remote_addr":"[fd00:1122:3344:101::b]:61433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.808597037Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62337"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.808819004Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9faf322b-8a02-4c1a-b9ac-057d9d7f0dbb","remote_addr":"[fd00:1122:3344:101::c]:62337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.82043811Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46913"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.820581724Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9d66df89-c7ca-40be-811c-683dd8887ff0","remote_addr":"[fd00:1122:3344:101::a]:46913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.867375119Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.86739757Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"04294c12-36c5-40b2-a7b7-390edbc60dcf","remote_addr":"[fd00:1122:3344:101::a]:46913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.871914841Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50149"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.872124467Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d7cda144-8b92-42f3-97ba-47df84c95716","remote_addr":"[fd00:1122:3344:101::a]:50149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.878358497Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"05a3df57-83cb-4f42-adb3-24074b7ae922","remote_addr":"[fd00:1122:3344:101::a]:46913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.87843071Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48530"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.878555973Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9c1f771a-dda2-4a9e-a2e0-422c93dfa40b","remote_addr":"[fd00:1122:3344:101::b]:48530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.883003042Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65036"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.883083684Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.920985352Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.975679296Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5970e6ed-cd02-41c6-b607-8a1f361e7671","remote_addr":"[fd00:1122:3344:101::c]:65036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92618,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.975800049Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50586"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.976009885Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.976133439Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:15.979688202Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1e60410d-3578-4594-bf97-14dbb5a3b83e","remote_addr":"[fd00:1122:3344:101::c]:65036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3658,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.014130259Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.07079525Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.070868572Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"33244eb4-8849-47a2-b87a-c8cd21f596c1","remote_addr":"[fd00:1122:3344:101::b]:50586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94915,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.074427226Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"be990ebe-dd7d-47dd-8ba9-f4cc9e51128a","remote_addr":"[fd00:1122:3344:101::b]:50586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":737,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.10601476Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.161974961Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f50233c3-fea0-4cb9-8490-e37c7e513a84","remote_addr":"[fd00:1122:3344:101::a]:58586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182084,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:16.162314981Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d7426309-5735-4278-9855-0031c0219a53","remote_addr":"[fd00:1122:3344:101::a]:58586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:23.531234004Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:23.531347908Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1cc0df6d-2e88-4fa4-acd1-af849049f8c0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39767","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.588311513Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51197"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.588480609Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"607d6768-72ad-487f-99ad-83a6156fdf73","remote_addr":"[fd00:1122:3344:101::b]:51197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.62132585Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.621426072Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"16565915-c018-41fc-a311-34c7375c807c","remote_addr":"[fd00:1122:3344:101::c]:51467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.629882997Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.629900148Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2e1bc087-4352-4817-a1da-9e340e373da8","remote_addr":"[fd00:1122:3344:101::b]:51197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.642002308Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.642182724Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2726d04a-aa90-4d9c-8158-5e23b58943fc","remote_addr":"[fd00:1122:3344:101::b]:51341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.64482053Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61102"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.644909012Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.690789051Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764241158Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2c71d5bc-71c9-4e60-a44f-3262e269ac99","remote_addr":"[fd00:1122:3344:101::b]:61102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119359,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764531767Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764577308Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b7e422bb-d7fe-4f24-9a45-90e9710f6ee4","remote_addr":"[fd00:1122:3344:101::c]:51467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":158,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764604899Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34356"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764682681Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47778"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764894807Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"addb2410-4d12-49bf-a401-1db6f470c5c1","remote_addr":"[fd00:1122:3344:101::b]:61102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":310,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.764926378Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.770284143Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"be6bcabd-79b6-4a89-a797-b79e6ccca9d9","remote_addr":"[fd00:1122:3344:101::a]:47778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5296,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.770430678Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41285"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.770439488Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"29e877c1-a8e9-45eb-8a3e-fc0660b8c637","remote_addr":"[fd00:1122:3344:101::b]:51197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5728,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.770774577Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"835839e0-563e-435f-a8a5-f2f3fb65531e","remote_addr":"[fd00:1122:3344:101::c]:41285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":308,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.776090451Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4501d3aa-e520-46f9-8f27-cea5b8116de1","remote_addr":"[fd00:1122:3344:101::c]:51467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":204,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.786815492Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35132"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.808911842Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.81334526Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.813412572Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1e744243-c934-428a-9b79-5faad180fff8","remote_addr":"[fd00:1122:3344:101::a]:47778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":558,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.822228258Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"60720c78-f208-4783-9697-f46ae12ffe3c","remote_addr":"[fd00:1122:3344:101::a]:47778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.87479916Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cc3ed664-b84b-44f3-ae58-1c0b360b91a3","remote_addr":"[fd00:1122:3344:101::c]:34356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109915,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.874881232Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.879314741Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e3ae69f3-ad8d-46df-9a10-07683f3c7ce0","remote_addr":"[fd00:1122:3344:101::c]:34356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":744,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.902696208Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.903081179Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"69ffe768-b159-4498-be49-9451355b9c85","remote_addr":"[fd00:1122:3344:101::a]:49089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":242,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.911097621Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.971313275Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e9751ef9-4e53-4d31-b9e5-2f438be1a77e","remote_addr":"[fd00:1122:3344:101::a]:35132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184303,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:45.971680275Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4f2159d9-2811-4da2-9a24-def040d6e849","remote_addr":"[fd00:1122:3344:101::a]:35132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:53.5633509Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:53:53.563444493Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"94a0304a-c327-4037-b6c0-8a4881a22956","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48935","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.570333699Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.570516145Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a7b0440b-ca34-4c45-9786-9671582bdbd4","remote_addr":"[fd00:1122:3344:101::a]:34470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.666791843Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.666822623Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"69070795-fb49-450b-91b1-3ef1571e1bed","remote_addr":"[fd00:1122:3344:101::a]:34470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.67775898Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6d47167d-8cd1-4c0a-a11a-2aa81a35e204","remote_addr":"[fd00:1122:3344:101::a]:34470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.710418296Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.710518099Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4db77610-b178-49a5-a543-71ca04dc8957","remote_addr":"[fd00:1122:3344:101::b]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.725314867Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.725332018Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"05dac0dd-6b0d-4bdf-86eb-d73806e907ba","remote_addr":"[fd00:1122:3344:101::b]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.78239361Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.782585296Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b14a5e17-873f-48f6-bcb9-4107899ea2f4","remote_addr":"[fd00:1122:3344:101::b]:60498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.785619804Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"82c49129-56a1-4917-89c5-d72b9206c44b","remote_addr":"[fd00:1122:3344:101::b]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.801793802Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.801870474Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"21600b5e-da64-423d-8d14-455657317c69","remote_addr":"[fd00:1122:3344:101::c]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.834016665Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41733"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.834114298Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.87734799Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.941348173Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c2e717e8-1eae-4831-8c16-7c5d45493889","remote_addr":"[fd00:1122:3344:101::b]:41733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107270,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.941628631Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.941639101Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.941652162Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"def6d9fc-b1cf-41f6-aa5a-370db3963415","remote_addr":"[fd00:1122:3344:101::c]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.941723794Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.94193587Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"38159d26-1c99-4334-a735-44b438f07672","remote_addr":"[fd00:1122:3344:101::b]:41733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.942002632Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0ed659b3-fcde-4198-a3df-bf3e0bc619f0","remote_addr":"[fd00:1122:3344:101::a]:47549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.942061404Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.951541688Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"35d29eb2-ebea-411e-be6c-36e8e3a3ff38","remote_addr":"[fd00:1122:3344:101::c]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":272,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.954688509Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56765"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:15.983353779Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.009227669Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.009598379Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3af965c4-c5de-46de-afc8-4462a479630c","remote_addr":"[fd00:1122:3344:101::c]:33431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.039368842Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.039423243Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b7dd865c-deee-4401-84a1-2ec2fbee9329","remote_addr":"[fd00:1122:3344:101::c]:64738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97461,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.043337096Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"93cfb758-ba7c-4846-a162-d92fdb520bd8","remote_addr":"[fd00:1122:3344:101::c]:64738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":816,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.077152686Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.134604309Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"207e6dc9-dba9-4a8a-a981-f31118793c2e","remote_addr":"[fd00:1122:3344:101::a]:56765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179854,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:16.13497747Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7d4ec631-076b-444e-9298-8aacde826231","remote_addr":"[fd00:1122:3344:101::a]:56765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:23.600646374Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:23.600742457Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0a9c0857-95ea-44ed-92f8-c11e9febdb8e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63078","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:24.955136978Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:24.957677422Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.010960125Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.011630254Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.032849929Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.039567514Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.040259484Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.063645591Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:25.069874441Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.271377244Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.651372039Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.653192031Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.686614019Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.713883989Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.850992569Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.958334977Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.958952945Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.959672276Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:34.960376396Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.012456395Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.013023651Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.013575287Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.034482662Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.041378592Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.041935448Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.065732777Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:35.07167691Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.274455579Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.65364481Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.655459793Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.689407366Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.716187202Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.853079345Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.960204987Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.961016321Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.961804783Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:44.962538955Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.014726686Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.015446317Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.016158148Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.036039173Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.042999875Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.043641904Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.067921917Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:45.072576351Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.577059031Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.577104162Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.577228646Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"06781725-60bf-4c77-af81-fd0746f92a83","remote_addr":"[fd00:1122:3344:101::b]:56397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.57739395Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"88bbea0d-c7ba-4fc8-86cb-82289a9b1114","remote_addr":"[fd00:1122:3344:101::c]:38474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.595936118Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.595962308Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c56c0764-e185-4ff2-aab4-d16f22d9b729","remote_addr":"[fd00:1122:3344:101::c]:38474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.604972599Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9662364e-2f75-497a-8585-1976cf4e5272","remote_addr":"[fd00:1122:3344:101::c]:38474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":204,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.605029501Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.605108683Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9688e5a0-94db-4009-9c33-75b74640e45e","remote_addr":"[fd00:1122:3344:101::a]:56858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.616297447Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.616466052Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"64e2cddf-77fe-471f-9e3c-24d4f0c9d5df","remote_addr":"[fd00:1122:3344:101::b]:36098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.635332088Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.635355189Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"502031d9-cfda-4755-8734-c6db496b068c","remote_addr":"[fd00:1122:3344:101::b]:56397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.644292808Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0d0c3412-6835-4ef5-a250-fbf46b8805c9","remote_addr":"[fd00:1122:3344:101::b]:56397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.650610381Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.650637162Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"59686502-3fa2-4b8d-a1df-d9fff1ebada5","remote_addr":"[fd00:1122:3344:101::a]:56858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.659649123Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0137ea99-7b98-4747-943e-6296cfb9a228","remote_addr":"[fd00:1122:3344:101::a]:56858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.688187329Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47162"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.68821355Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62681"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.688305312Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.694482361Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.694914954Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6fef563f-5451-4163-8624-3cfdf4aae9e1","remote_addr":"[fd00:1122:3344:101::c]:50380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.699026403Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63389"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.708774515Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.709138236Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"75fb9750-b47e-46a2-998f-e24ac8865381","remote_addr":"[fd00:1122:3344:101::a]:50142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":213,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.73034228Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.789020659Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.789085961Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"435773ce-a42f-4b4d-987b-308b4deb33d1","remote_addr":"[fd00:1122:3344:101::a]:47162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100808,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.793577581Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f4e79bac-8079-4849-a4a6-6abfd4f32ea5","remote_addr":"[fd00:1122:3344:101::a]:47162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":716,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.830078527Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.888363965Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.888426187Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"4f0fed12-c11d-434e-9a6d-1fcc58e174d4","remote_addr":"[fd00:1122:3344:101::c]:62681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200060,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.893655298Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"57735447-b9fa-4f7f-8652-c0a99a322993","remote_addr":"[fd00:1122:3344:101::c]:62681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":703,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.923559944Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.979087822Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"827f463c-aa76-4452-ab81-5661f8e5fcb9","remote_addr":"[fd00:1122:3344:101::b]:63389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280000,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:45.979438983Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4de129a0-6546-47e5-a5de-bad11d95a411","remote_addr":"[fd00:1122:3344:101::b]:63389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:53.635837271Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51636"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:54:53.635964385Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"d5996456-b08e-465e-acdf-25808e30f0f6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51636","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.278206723Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.655589041Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.657376933Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.692461239Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.71837106Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.855023667Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.96215697Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.962777217Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.963506598Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:54.964064505Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.020909861Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.021678903Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.022392264Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.037672886Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.044501844Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.045244216Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.069994482Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:54:55.073744921Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.281953938Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.657486483Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.658281386Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.695265347Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.720082716Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.857129864Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.964315638Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.964935626Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.965656197Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:04.966205622Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.023069169Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.023662726Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.024188272Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.040030271Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.045985213Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.046597981Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.071822401Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:05.075733714Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.285576176Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.659661469Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.660518324Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.697851235Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.721711476Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.858372113Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.965468274Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.967143363Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.967859484Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:14.968592105Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.024850674Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.025385539Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.025935585Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.041762734Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.047809479Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.048331664Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.074308676Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:15.078012354Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.708689997Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59913"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.708855972Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cc8187c9-4957-40a7-b9c8-43e74f8e9200","remote_addr":"[fd00:1122:3344:101::b]:59913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.746825792Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42659"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.746964696Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6e76ceea-90d5-4556-800c-e3b0f0cc1bd0","remote_addr":"[fd00:1122:3344:101::c]:42659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.761231689Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.76125916Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e21951f4-db6c-4226-9f0f-9d405309a106","remote_addr":"[fd00:1122:3344:101::b]:59913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.772293449Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5012ee0c-364e-47ea-8e01-d3a1d73d1f7e","remote_addr":"[fd00:1122:3344:101::b]:59913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.797388266Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.797408476Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ed0a2a10-a753-4f83-8251-ffa87ca5d852","remote_addr":"[fd00:1122:3344:101::c]:42659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.808449496Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"39a3a12e-b969-437f-9e1e-113408125608","remote_addr":"[fd00:1122:3344:101::c]:42659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.845021225Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.845110078Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0cfc294a-df7d-41d9-9ad4-353863902c0f","remote_addr":"[fd00:1122:3344:101::a]:47136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.862582534Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.862604495Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"36e36fb9-c818-4969-a3db-598404830515","remote_addr":"[fd00:1122:3344:101::a]:47136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.868272549Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.868530926Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"7ab89b84-0d9a-4c68-8bdb-a944d3c3fabc","remote_addr":"[fd00:1122:3344:101::c]:53414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.873305164Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"de8d1d9f-7f31-4a6c-8dd4-ce705e289501","remote_addr":"[fd00:1122:3344:101::a]:47136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.884050736Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59888"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.884222521Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:15.935134575Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.003603808Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d4c8f961-72a5-4199-a12d-98438a84f593","remote_addr":"[fd00:1122:3344:101::c]:59888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119437,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.003847295Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41737"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.003913617Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42650"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.003960898Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62476"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.004000799Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.004010879Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ac0cb3cb-14ba-44fb-9a1a-0ed9c6e209c9","remote_addr":"[fd00:1122:3344:101::c]:59888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.004205285Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.008453968Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"5f1a0d99-d3a1-4739-8bd8-e82392002aa5","remote_addr":"[fd00:1122:3344:101::a]:37970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.008608463Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c94e5533-333a-41d0-bf28-53fbfafb634a","remote_addr":"[fd00:1122:3344:101::b]:41737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4305,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.041189326Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.093311965Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.093385698Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5b82bbab-830a-451d-8da4-142301a5f2d9","remote_addr":"[fd00:1122:3344:101::b]:62476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89245,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.096831958Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bb154e5b-4ff5-427d-8396-f99e9c445441","remote_addr":"[fd00:1122:3344:101::b]:62476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":556,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.126094425Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.176838515Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8bb0cb4c-ff47-4fb5-abf0-49ffdc9e40cf","remote_addr":"[fd00:1122:3344:101::a]:42650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172562,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:16.177251687Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bffbd80a-b468-4b25-88d6-9dfff3aea720","remote_addr":"[fd00:1122:3344:101::a]:42650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:23.669316445Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33377"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:23.669408378Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bc300872-276f-4fe5-ab0f-203f211f430c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33377","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.289493954Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.662005642Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.662756034Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.708938981Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.723493402Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.860377796Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.966719715Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.968497657Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.969214388Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:24.970967639Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.027064373Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.027792364Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.028363801Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.043584311Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.049700568Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.05044435Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.075225878Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:25.080248273Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.293631599Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.663704996Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.66453532Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.71150302Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.725430823Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.862656217Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.96947885Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.970261933Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.970995184Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:34.972746335Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.028761907Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.029476638Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.030185638Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.045920274Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.052065232Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.052575936Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.076718566Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:35.081502944Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.297078353Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.665136571Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.665927714Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.71480155Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.727981571Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.864060212Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.972131222Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.972739419Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.97344699Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:44.974006576Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.031297565Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.031866252Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.032384687Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.047235117Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.054180608Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.054974271Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.079241364Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:45.083338122Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.562361124Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38646"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.562524779Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"deb5d074-7e96-4817-b122-d2f8a337d132","remote_addr":"[fd00:1122:3344:101::a]:38646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.57741773Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.577439221Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8152cd30-8efe-49c4-b90b-936e87db4474","remote_addr":"[fd00:1122:3344:101::a]:38646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.598334706Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.59848004Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e394f19c-d428-48dd-a4d5-20fc18bed89a","remote_addr":"[fd00:1122:3344:101::c]:36491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.608193902Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.608385937Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"24c1c102-7b6f-4d63-8cb3-f1004dae6c97","remote_addr":"[fd00:1122:3344:101::a]:43620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.613083563Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.613100613Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e10271dd-749d-42f3-8544-822d78c358bd","remote_addr":"[fd00:1122:3344:101::c]:36491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.615178754Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b9be1e30-3bde-47c9-b857-a8881ecee7fc","remote_addr":"[fd00:1122:3344:101::a]:38646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.621050754Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c7a5b84e-4921-41cc-ab09-8f6d3616c327","remote_addr":"[fd00:1122:3344:101::c]:36491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.630630081Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51045"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.630719504Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.670528137Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.755395194Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"32b0c1dd-bc1c-4fad-b3cf-22fbb337a3f9","remote_addr":"[fd00:1122:3344:101::a]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124704,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.755568969Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43954"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.755628571Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37118"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.755692753Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.75593079Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f2a1f4a0-b4af-4318-b447-ceca4a2f3d2e","remote_addr":"[fd00:1122:3344:101::a]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":276,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.755960171Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.756041753Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b20c3a5e-c350-4c9d-a441-2c201269f80c","remote_addr":"[fd00:1122:3344:101::b]:37118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.762646884Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a88fbbe3-4bb2-4dce-b88c-0b5624fc6996","remote_addr":"[fd00:1122:3344:101::c]:43954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6693,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.807971616Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.813533367Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53752"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.81362534Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.813650511Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2e1b56f8-5b18-4c76-af41-07bcfcce5947","remote_addr":"[fd00:1122:3344:101::b]:37118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":741,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.822749794Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5576f508-bd87-4244-8eb9-1c01b770f141","remote_addr":"[fd00:1122:3344:101::b]:37118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":314,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.876804829Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.876895462Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f50d57b4-dc25-44e1-b77e-a6b8bb7c5278","remote_addr":"[fd00:1122:3344:101::c]:36347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121031,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.881895247Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bdd39984-fe29-44e5-b5b5-7bd8a63f651e","remote_addr":"[fd00:1122:3344:101::c]:36347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":815,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.907828978Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60804"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.908218659Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e32390d1-d26f-4c8e-b580-2b551eb29656","remote_addr":"[fd00:1122:3344:101::b]:60804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.917180339Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.977062423Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a3ac7527-3a7e-4f6b-a773-c8c472274279","remote_addr":"[fd00:1122:3344:101::b]:53752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163328,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:45.977430803Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6ffc1f2a-5047-404e-9e52-00685505be78","remote_addr":"[fd00:1122:3344:101::b]:53752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:53.70597863Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:55:53.706116974Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"730d4bfc-804a-4c99-ad79-34e97b563884","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60802","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.301907357Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.667607227Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.668538464Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.717869962Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.731419455Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.865846857Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.97369588Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.974298577Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.975015668Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:54.975726309Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.03379616Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.034325026Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.034878342Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.048600039Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.05517199Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.056751645Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.081196613Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:55:55.084824018Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.305251219Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.670264719Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.671122534Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.720289388Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.733053888Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.867449049Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.975648962Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.976293051Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.977033893Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:04.977807865Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.035667891Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.036215876Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.036751382Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.050681915Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.056636588Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.058272635Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.083535217Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:05.086510323Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.309020484Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.672363446Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.673123998Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.723105696Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.734874827Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.868727592Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.977613205Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.978389118Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.97915171Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:14.979905092Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.03715416Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.038042035Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.038757226Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.052873195Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.05788556Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.060560537Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.085387526Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:15.087989662Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.69809497Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.698260165Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"96866aef-1332-485c-8ec3-f2a385da5264","remote_addr":"[fd00:1122:3344:101::c]:54301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.716712989Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.71673284Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bf5b1594-5b47-4437-a2b5-5a577346adab","remote_addr":"[fd00:1122:3344:101::c]:54301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.729480149Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"959106fd-f43d-4677-b428-2698c829563c","remote_addr":"[fd00:1122:3344:101::c]:54301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.804416449Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.804494881Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"091c1d1a-5260-416c-95b3-208e537dc457","remote_addr":"[fd00:1122:3344:101::b]:52145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.839623228Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.839918507Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"35848463-9578-4d98-8269-f3bd742d55c6","remote_addr":"[fd00:1122:3344:101::a]:42621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.856222599Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.856409044Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"f0ec1ec4-2618-499f-93c8-e7aab135b204","remote_addr":"[fd00:1122:3344:101::c]:40086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.877750172Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.877812794Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ac92001e-eae2-4a32-a8cb-43c9dcb35fbc","remote_addr":"[fd00:1122:3344:101::b]:52145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.887507475Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.887627029Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5b6cc36c-627d-459b-930d-1946ef9bc0f5","remote_addr":"[fd00:1122:3344:101::a]:54282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.905857676Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.905890787Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"30751b1e-4a23-4a39-83b1-0e65fe2e8e42","remote_addr":"[fd00:1122:3344:101::a]:54282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.943304081Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55364"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.943436415Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:15.998228721Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069347341Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cf55fe4b-447e-4782-9150-a59cb8a4900e","remote_addr":"[fd00:1122:3344:101::c]:55364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125947,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069639189Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49510"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069737062Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50698"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069835655Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50936"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069880146Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069907427Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"14174edb-0450-4f16-8ca7-4fd5c36614ad","remote_addr":"[fd00:1122:3344:101::c]:55364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069943388Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f34b2355-db46-4b4f-8122-1a6c560553ec","remote_addr":"[fd00:1122:3344:101::b]:52145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":406,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.069982939Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60358"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070050741Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53976"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070280438Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070327929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.0703433Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070410082Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9b97acc5-7ed5-40b8-85d9-a66daf016868","remote_addr":"[fd00:1122:3344:101::a]:54282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":657,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070433022Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d1712b6f-99a5-4351-a362-d2379b24d911","remote_addr":"[fd00:1122:3344:101::b]:49510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":372,"response_code":"200"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070464193Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.070482854Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077177948Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077554689Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077712133Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077800926Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077842497Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.077891088Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.0779335Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.078123145Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.078148406Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"cb9e77a9-657f-4bdc-89f5-0a76e842f6b7","remote_addr":"[fd00:1122:3344:101::c]:50936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7823,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.093296214Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.093759218Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.112341786Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.112435359Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.118076192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.123250242Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.128946737Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.129185054Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.129998087Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41727"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.13595212Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136075783Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136087024Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136111994Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136149006Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136158896Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136176296Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.136214437Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8788d727-f912-4bfc-9fd7-cacdcca3c596","remote_addr":"[fd00:1122:3344:101::a]:41727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":614,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.146945758Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.146989799Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.17877458Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.179007027Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193175057Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57698"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193724863Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32de9250-63c9-46be-9a6b-ee4af19ab1ab, kind: Service { id: 48221b8a-d832-4d05-ae47-e92f81388c0c }, name: Name(\"external-dns-48221b8a-d832-4d05-ae47-e92f81388c0c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 243, 126])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: da2f10b4-11b4-4931-a7f7-d415353256ec, kind: Service { id: 8c0d54f6-76b3-47cc-9828-b8e677cc2961 }, name: Name(\"external-dns-8c0d54f6-76b3-47cc-9828-b8e677cc2961\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 129])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 68cb12a7-f65f-4940-ad7a-12ac0e8fa220, kind: Service { id: 7d1b51fe-adf1-4350-9c25-9240054e3911 }, name: Name(\"nexus-7d1b51fe-adf1-4350-9c25-9240054e3911\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 76])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a5946ca8-a389-4a86-aed2-97d4e8639c67, kind: Service { id: 13db289d-f9b6-41e9-b3bd-5ab4f3105694 }, name: Name(\"nexus-13db289d-f9b6-41e9-b3bd-5ab4f3105694\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 142])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b897d57d-b436-467c-89e9-441e43e441fa, kind: Service { id: 79994e02-6747-4071-83af-7872b31640fb }, name: Name(\"nexus-79994e02-6747-4071-83af-7872b31640fb\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 66])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193773104Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193792225Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193821796Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193840506Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193860397Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193866667Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.193887127Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ec2e74b5-03a8-4659-9bdb-cc89649751d5","remote_addr":"[fd00:1122:3344:101::b]:57698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":531,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.194901077Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.194908807Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.20501395Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.210768676Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"4257014d-0912-4451-b237-fb8faa7d15d8","remote_addr":"[fd00:1122:3344:101::b]:53976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140405,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.211318392Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0f914f98-7768-4d54-9a1e-40c88a977d9c","remote_addr":"[fd00:1122:3344:101::b]:53976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.216490152Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.216939045Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.233659759Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.233711591Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.256328646Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.261284999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.261565247Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.276387917Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.276475819Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.294507451Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.294841881Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.308572129Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.30862595Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.330298858Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dbc1cc68-99a2-4259-b62a-a5908910d5f5","remote_addr":"[fd00:1122:3344:101::a]:65021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259951,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.330747111Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"10665143-4e2f-4821-a490-d204c950521d","remote_addr":"[fd00:1122:3344:101::a]:65021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":236,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.333061598Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.333087009Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.346172877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.346243819Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.363899611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.364164798Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.377889336Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.377932327Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.397527645Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.397570286Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.408444131Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.408510233Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.412035875Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"c7f95f0e-39e5-4712-bd74-8f31a1784e76","remote_addr":"[fd00:1122:3344:101::a]:50698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":341924,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.428235074Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.428288795Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:16.441797837Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"7eaee198-2f53-4ab1-8a21-94ea6c4fdb44","remote_addr":"[fd00:1122:3344:101::b]:60358","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":371551,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.066480326Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39588"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.06662287Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.066641521Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.066680082Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.06695133Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b55e819d-82d6-47a9-8489-82b572951fbc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.081332646Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.081378068Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_b55e819d-82d6-47a9-8489-82b572951fbc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: ba8497a2-b0b3-4f86-b2e2-626db79a155f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.105675141Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.105736523Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_ba8497a2-b0b3-4f86-b2e2-626db79a155f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d0e3c514-0a4e-463a-96d3-0dd3fac40f52 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.130609413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.130666155Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d0e3c514-0a4e-463a-96d3-0dd3fac40f52"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 42743ca1-d71f-4721-aaf3-5ac7d5df758b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.154800874Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.154849085Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_42743ca1-d71f-4721-aaf3-5ac7d5df758b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3b09028c-7f4f-4a0f-bbae-e1491d795acc (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.176212184Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.176262705Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_3b09028c-7f4f-4a0f-bbae-e1491d795acc"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.199054526Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.199100607Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.220753944Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.220809696Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 890a5a3b-e222-4b29-81b1-a24f27413d31 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.242228016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.242278527Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_890a5a3b-e222-4b29-81b1-a24f27413d31"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 999a4caf-5d6a-4280-bda0-f8e55228f7be (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.263188543Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.263240554Z","hostname":"sock","pid":1572,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1019","zpool":"oxp_999a4caf-5d6a-4280-bda0-f8e55228f7be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:17.275115398Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"c58e3a83-061f-4e51-a26e-c70089638174","remote_addr":"[fd00:1122:3344:101::c]:39588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":208560,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:23.738457067Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:23.738616682Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f8fe4709-4457-40c5-ab73-1d64e78570ca","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42170","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.313583002Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.673978789Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.675889824Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.724767729Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.736639563Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.870215891Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.979049652Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.97963932Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.98139017Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:24.981945227Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.039176574Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.039830183Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.040380859Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.05526824Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.058921396Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.061700156Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.087183334Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:25.089045008Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.31826162Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.676976119Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.677756021Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.726586345Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.738434778Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.872102448Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.981166777Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.981771544Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.982515006Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:34.984116722Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.040891636Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.041425852Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.041980718Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.056981792Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.060893476Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.062444101Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.089648878Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:35.091313706Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.321535889Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.680742261Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.681517184Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.728449503Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.740314527Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.873987367Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.983095316Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.983740935Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.984460396Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:44.986190916Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.042973401Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.043488465Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.04400852Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.059140339Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.063126954Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.063819424Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.091183637Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:45.092706181Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.485697811Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:32794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.485827035Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9845a3c0-eaf1-47cb-93cf-ee00b3927dfd","remote_addr":"[fd00:1122:3344:101::c]:32794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.519176701Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61760"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.519312035Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"65e61b2a-9848-4c2b-91b2-891a204bf9dd","remote_addr":"[fd00:1122:3344:101::b]:61760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.542334311Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.542380873Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4c8d248-b090-48e0-80ba-3bef9c778b9e","remote_addr":"[fd00:1122:3344:101::c]:32794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.55438112Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.554610917Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cb648b6e-1c6b-41a0-b901-120272484174","remote_addr":"[fd00:1122:3344:101::a]:39307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.556742189Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"438bdac1-ab6f-45f3-8a33-c100d65ad43f","remote_addr":"[fd00:1122:3344:101::c]:32794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.565306137Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.565321527Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8aac57f2-cfcc-4793-b2ff-a76d5ed8643d","remote_addr":"[fd00:1122:3344:101::b]:61760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.58129944Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8532d37a-199e-442a-a737-7308aadfeec8","remote_addr":"[fd00:1122:3344:101::b]:61760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.599990761Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.600005032Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"78db5074-fb7d-47bf-a770-20f538bf0e7b","remote_addr":"[fd00:1122:3344:101::a]:39307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.608068925Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a205fc7b-e3db-4dda-8b61-6039b4f30b69","remote_addr":"[fd00:1122:3344:101::a]:39307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.656820357Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.657008392Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4d8f90bf-2d5e-4f09-9506-0d1196dd572e","remote_addr":"[fd00:1122:3344:101::c]:61982","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.658180266Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.65830626Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"338194fb-8c74-4a44-a004-dc41f9afd755","remote_addr":"[fd00:1122:3344:101::b]:51783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.664914181Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.665025264Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"fbf406ca-44d9-4b6b-8b75-79b5c7c95971","remote_addr":"[fd00:1122:3344:101::a]:44354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.672232003Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58221"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.672307966Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.711659135Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.772970931Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8027b713-f820-40c4-87ae-ea67f2ed0666","remote_addr":"[fd00:1122:3344:101::b]:58221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100698,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.773084114Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.773107145Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45750"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.773335091Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.777048669Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3e1d6fa6-7047-4b50-979e-e42fd18bd90b","remote_addr":"[fd00:1122:3344:101::b]:58221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3697,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.807956653Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.86414527Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.864233773Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"df52f298-cfe0-4890-9516-be7a461061bc","remote_addr":"[fd00:1122:3344:101::a]:45750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90981,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.868327631Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"15e4e7ba-2b77-4442-b394-a281cec4af7b","remote_addr":"[fd00:1122:3344:101::a]:45750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":391,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.900437061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.957063611Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7c223c95-f0c5-46c5-a341-51be0c4f34b8","remote_addr":"[fd00:1122:3344:101::c]:55199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183636,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:45.957449142Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"715bb54d-4eae-41b6-9785-5cad7519d85f","remote_addr":"[fd00:1122:3344:101::c]:55199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:53.774294647Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:56:53.774426681Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"12413636-d73d-4512-bad3-3d6704c7da6d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34735","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.325131888Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.684778303Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.685519025Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.729399516Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.742245078Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.876350771Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.98543806Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.986191551Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.986923433Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:54.987633553Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.044997214Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.045717045Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.046424156Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.061508563Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.06522008Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.065776936Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.092762248Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:56:55.093310774Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.328730137Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.688251469Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.688962519Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.730956105Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.743782427Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.878375964Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.995082013Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.995812094Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.996520695Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:04.997210565Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.046279586Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.048024566Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.048710866Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.062501556Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.066227033Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.067764678Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.094730869Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:05.095412289Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.332715445Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.69096912Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.691705071Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.732467172Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.745213671Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.879948682Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.997139715Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.997699142Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.998427853Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:14.998937458Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.04806189Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.049582884Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.050095899Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.065230307Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.068312996Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.069868272Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.097121731Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:15.097634086Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.599203201Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63024"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.599420777Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fa45e078-bec1-4500-aa35-9d68d28b28ec","remote_addr":"[fd00:1122:3344:101::a]:63024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.6737705Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.673803751Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d3e2c8ee-b31b-4994-b2b9-7431fb367544","remote_addr":"[fd00:1122:3344:101::a]:63024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.746137426Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.74627355Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"df893fdf-312f-45a4-ad18-ce551ae6e3c3","remote_addr":"[fd00:1122:3344:101::b]:54741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.757536546Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.757823314Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"96397c79-bc96-49ef-a84c-6f980394096a","remote_addr":"[fd00:1122:3344:101::b]:45979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.784176407Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.784284881Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fe99b66d-3fa7-4745-9304-3976a09ec917","remote_addr":"[fd00:1122:3344:101::c]:57937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.804923277Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.804943348Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ec9dbcec-03e8-4f72-9b5d-85796521218c","remote_addr":"[fd00:1122:3344:101::b]:54741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.816643697Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8249c507-7350-44dd-a52d-f9c76466e973","remote_addr":"[fd00:1122:3344:101::b]:54741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.844752311Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.844771081Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b3fe3aa5-6e4d-403f-9c7b-34d42d182ab8","remote_addr":"[fd00:1122:3344:101::c]:57937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.856317946Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c2672c07-f283-4c4d-b512-79458cf171f0","remote_addr":"[fd00:1122:3344:101::c]:57937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.857038696Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.857222922Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"eef6f8c5-3bba-490c-a4a8-f54e56e0240b","remote_addr":"[fd00:1122:3344:101::a]:45619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.868491378Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"790e31cd-2f43-4391-94ef-7ef03c3efa65","remote_addr":"[fd00:1122:3344:101::a]:63024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.886445908Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46551"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.88653085Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:15.943392847Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.00600169Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e21b1209-fa36-4b7e-9fa8-57aafa26fca1","remote_addr":"[fd00:1122:3344:101::b]:46551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119512,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.006124074Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60229"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.006157995Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38896"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.006217256Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45140"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.006447843Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.010171211Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"639a428f-d98f-4b1d-8a35-7e7011053a7d","remote_addr":"[fd00:1122:3344:101::b]:46551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3680,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.01048048Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"accb741e-9395-45a6-b5ad-3d74ba8ad709","remote_addr":"[fd00:1122:3344:101::c]:38896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3946,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.039459419Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.092796384Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.092890706Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"38e5e057-2424-41ef-a044-4a704b47af2b","remote_addr":"[fd00:1122:3344:101::a]:45140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86550,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.096552483Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"83372d98-9c45-4451-a75b-be3c61e1a521","remote_addr":"[fd00:1122:3344:101::a]:45140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":246,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.128090446Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.189048361Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"579290ff-9777-478e-97e2-16abe8bef96b","remote_addr":"[fd00:1122:3344:101::c]:60229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182614,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:16.189403981Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c1b9d9ad-e7d8-4129-bb15-e2127fbe02f3","remote_addr":"[fd00:1122:3344:101::c]:60229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:23.808072584Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:23.808217959Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4d2f1bd9-b31e-4010-938a-03cb7357b52b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51948","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.337424894Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.694185076Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.694928927Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.73508191Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.746964534Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.881736856Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.998547009Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.999102365Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:24.999782165Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.000475445Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.049830484Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.051497262Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.052053338Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.066910818Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.069745171Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.071420149Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.098184424Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:25.099726419Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.341668322Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.697803935Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.698541816Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.736330141Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.748140323Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:34.883522083Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.000738407Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.001390406Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.002085366Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.002783086Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.051581619Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.053110023Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.053611928Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.068374106Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.071089564Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.072696581Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.099441525Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:35.101020161Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:39.610993553Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.345062055Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.700936481Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.701680742Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.73856528Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.749340302Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:44.885137024Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.002821802Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.003592254Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.004272024Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.004951354Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.053981564Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.054724305Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.055411615Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.07077493Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.072299634Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.07389941Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.101630663Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:45.102353054Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.494653435Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.49484386Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ab66c64d-caf9-40da-b92f-79cda618b47a","remote_addr":"[fd00:1122:3344:101::c]:49212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.558826984Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.558857004Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4be68c3e-88c5-4cc2-a035-2c77d82520b1","remote_addr":"[fd00:1122:3344:101::c]:49212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.570691837Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6469a7a1-f45d-4bd9-9155-1e270102b20b","remote_addr":"[fd00:1122:3344:101::c]:49212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.572561581Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.572673844Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a0d61335-18bc-4293-b8a6-def456fa9613","remote_addr":"[fd00:1122:3344:101::a]:53638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.633235738Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.633271859Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e3499557-4c6a-4cb3-861c-5935da96ad91","remote_addr":"[fd00:1122:3344:101::a]:53638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.640695384Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.640783357Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"dab49848-e54d-4221-8616-3e9df762826d","remote_addr":"[fd00:1122:3344:101::b]:56835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.642645571Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.642880227Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e9019985-89ba-425a-b8f3-64facc9ea9e7","remote_addr":"[fd00:1122:3344:101::c]:61252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.658396177Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.658416048Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"46eebde9-252f-4667-a72e-61f9314ca399","remote_addr":"[fd00:1122:3344:101::b]:56835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.67059038Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5f283c20-4036-4299-8f11-8b287e8ce525","remote_addr":"[fd00:1122:3344:101::b]:56835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.687742197Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53536"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.68784468Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.749218597Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843212239Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0a0dd48a-c1c8-448c-af8f-9f64f515bfb4","remote_addr":"[fd00:1122:3344:101::c]:53536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155403,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843398154Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55833"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843463546Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38002"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843490977Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33072"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843551309Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36022"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843903579Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3936dd2e-3197-4b57-91a6-cf0c3aac4b83","remote_addr":"[fd00:1122:3344:101::c]:53536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":372,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.843975861Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.851235301Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3ee4ab46-6b0a-4978-bd3d-9d3f54560d88","remote_addr":"[fd00:1122:3344:101::b]:33072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7324,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.851396796Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"5cbd61e2-72d5-47ff-8b39-7df01b95d070","remote_addr":"[fd00:1122:3344:101::a]:55833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7465,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.851425047Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1a50bd3b-cfd9-4596-8a6c-27565a679c0e","remote_addr":"[fd00:1122:3344:101::a]:53638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7854,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.891719034Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.955911812Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"15b2c77c-28ba-467c-9c33-222aa6f2493a","remote_addr":"[fd00:1122:3344:101::a]:36022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112143,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.955986315Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.960392782Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c68c2fac-ee66-4c38-b7bb-570a14f524ac","remote_addr":"[fd00:1122:3344:101::a]:36022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":739,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:45.995630323Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:46.059791501Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"634049ce-457f-4e1b-ae8c-7e0d62f1ca7d","remote_addr":"[fd00:1122:3344:101::b]:38002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":215887,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:46.060193852Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"be7525e6-d619-4398-90aa-69c949dfbf4d","remote_addr":"[fd00:1122:3344:101::b]:38002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.106872389Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:32838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.107074205Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0be9af3c-83a7-4242-9833-4a4743f354c2","remote_addr":"[fd00:1122:3344:101::a]:32838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.107808136Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(2965576) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.107843167Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"PUT","req_id":"26246483-87f9-499a-9181-eb82b675cf80","remote_addr":"[fd00:1122:3344:101::a]:32838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":587,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.201421537Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52864"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.213577219Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 992dae43-7740-49b5-814b-c5341916fc63, project_id: cc48da91-268d-48d2-b3ae-6259a7958913 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-14T04:57:49.201228392Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(2965576))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-a3f0e43214b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2c7ab8d8-cf73-41c6-b320-9db510c70ee6, read_only_parent: Some(Volume { block_size: 512, id: 81cd08b2-cc40-4fae-8b81-159da6842ebc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 65537679-e553-4001-8693-e674cb8aa5d0, key: Some(\"oTPxOzv+M/aTxEW5OqJ0DFAAmkeyNpwB4iCOOh1u4U8=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2c7ab8d8-cf73-41c6-b320-9db510c70ee6, key: Some(\"BZz/odnerjbqaCvAYH4h4yH0tGmX7pT3tNuSHiicw1o=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-a3f0e43214b\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.2136095Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.213635131Z","hostname":"sock","pid":1572,"propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(2965576))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 957e93db-0834-4463-8104-342e752f7f7e, kind: Instance { id: f872674e-3abd-4856-abe7-9936da747fcf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(2965576), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-a3f0e43214b\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 2c7ab8d8-cf73-41c6-b320-9db510c70ee6, read_only_parent: Some(Volume { block_size: 512, id: 81cd08b2-cc40-4fae-8b81-159da6842ebc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 65537679-e553-4001-8693-e674cb8aa5d0, key: Some(\"oTPxOzv+M/aTxEW5OqJ0DFAAmkeyNpwB4iCOOh1u4U8=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2c7ab8d8-cf73-41c6-b320-9db510c70ee6, key: Some(\"BZz/odnerjbqaCvAYH4h4yH0tGmX7pT3tNuSHiicw1o=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-a3f0e43214b\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-14T04:57:49.201228392Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.214523457Z","hostname":"sock","pid":1572,"uri":"/vmms/2c73ff36-1861-4bac-ad42-72e89f1dbf2d","method":"PUT","req_id":"6f34932d-6fe8-415a-a4b4-501675a2969b","remote_addr":"[fd00:1122:3344:101::a]:52864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":13038,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.303786912Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.305014077Z","hostname":"sock","pid":1572,"uri":"/vmms/2c73ff36-1861-4bac-ad42-72e89f1dbf2d/state","method":"PUT","req_id":"4704d898-73a7-473c-b00c-c683434ad5e2","remote_addr":"[fd00:1122:3344:101::a]:58675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1153,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.307064287Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 2965576 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.307089367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.347215569Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.370971127Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.385280481Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.385466177Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e8034723-82f4-46ae-8d30-aec0ce55aca5","remote_addr":"[fd00:1122:3344:101::a]:40033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:49.404359694Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {957e93db-0834-4463-8104-342e752f7f7e: {10.151.2.175: {00e47d94-6696-4bbb-afc0-b4f6e0bc322f}, 10.151.2.176: {00e47d94-6696-4bbb-afc0-b4f6e0bc322f}}} }"} {"msg":"Profile for oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:50.921400125Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:50.922436755Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 57.736901ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:51.114932069Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 108.343473ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:51.196780179Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 132.520783ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:51.339615476Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 369.760513ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:51.485230133Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 1.102262607s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:51.872216649Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 577.635107ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:52.989701889Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 628.947176ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:53.57952348Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:53.839521859Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:53.839683793Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3e2117f5-6d16-482f-9ae7-558ff22c518e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38345","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 1.074666559s","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.218695719Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.348267461Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.703576531Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.704290922Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.740104228Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.750971213Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:54.887557088Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.005923476Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.006528573Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.007220093Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.007755909Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.055596544Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.056128679Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.057665334Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.072421621Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.073993807Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.075547742Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.103313936Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.103832421Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 611.168643ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.305910573Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:55.951180079Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 59.089493ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:55.962308022Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 142.349685ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:56.03477274Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 114.79259ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:56.19533758Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\") failed: Property not found. retry in 520.349832ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:57:56.321272877Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:56.853771177Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:56.89048458Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.02790359Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.028896879Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.02895299Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5831781b-30ab-4415-9ed9-877ec57faaed","remote_addr":"[fd00:1122:3344:101::a]:40033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7626132,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-11-14T04:57:57.029895537Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.030085093Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.030099643Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-14T04:57:57.029895537Z }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.031188695Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-14T04:57:57.029895537Z }, migration_in: None, migration_out: None }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-11-14T04:57:57.090671688Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.090689328Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.090706338Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-14T04:57:57.090671688Z }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.090713339Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-14T04:57:57.090671688Z }, migration_in: None, migration_out: None }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:57:57.104513958Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5981d8ac-87b6-4982-bfdd-f03426c8095b","remote_addr":"[fd00:1122:3344:101::a]:40033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":349,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.351851721Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.706694667Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.707444389Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.742231386Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.752109442Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:04.888838992Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.006724186Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.008327892Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.009040493Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.009717812Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.057537127Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.058067122Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.059622337Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.074325073Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.074839178Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.077816064Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.104683192Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:05.10531135Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.902253378Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56721"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.9030301Z","hostname":"sock","pid":1572,"uri":"/vmms/2c73ff36-1861-4bac-ad42-72e89f1dbf2d/state","method":"PUT","req_id":"87211704-db21-4e3f-94df-e7409a25f471","remote_addr":"[fd00:1122:3344:101::a]:56721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":670,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-11-14T04:58:06.903144453Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.903166814Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.903186815Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-14T04:58:06.903144453Z }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.903203145Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-14T04:58:06.903144453Z }, migration_in: None, migration_out: None }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-11-14T04:58:06.912998139Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.913010359Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.913023059Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-11-14T04:58:06.912998139Z }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.91302908Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.91305494Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:06.913149543Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"7498626804358108963"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:06.913205325Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"15485300109431362625"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.913273367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:06.913284647Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/57ff02a4-a47e-49dd-bdd2-e8b99a648043/debug/bundle/zone\", \"/pool/int/8036e7f6-3d88-4263-96f1-ac55d658c5c4/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/b55e819d-82d6-47a9-8489-82b572951fbc/crypt/debug\", \"/pool/ext/ba8497a2-b0b3-4f86-b2e2-626db79a155f/crypt/debug\", \"/pool/ext/d0e3c514-0a4e-463a-96d3-0dd3fac40f52/crypt/debug\", \"/pool/ext/42743ca1-d71f-4721-aaf3-5ac7d5df758b/crypt/debug\", \"/pool/ext/3b09028c-7f4f-4a0f-bbae-e1491d795acc/crypt/debug\", \"/pool/ext/d5ec0ee3-bbc0-4fc4-b92f-6c8fc22d4e7b/crypt/debug\", \"/pool/ext/2f6cc1da-3ca4-4de2-85a8-05dd33e8db2d/crypt/debug\", \"/pool/ext/890a5a3b-e222-4b29-81b1-a24f27413d31/crypt/debug\", \"/pool/ext/999a4caf-5d6a-4280-bda0-f8e55228f7be/crypt/debug\"] }","zone_name":"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:07.572433946Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d\", bundle_id: dcd08e75-0607-4dd0-bd82-776261b9debf }, time_created: 2024-11-14T04:58:06.914020128Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_2c73ff36-1861-4bac-ad42-72e89f1dbf2d","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:07.572493617Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.326731838Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.345571403Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(2965576), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.345621375Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(2965576), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.345990795Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-14T04:58:09.345955905Z }, migration_in: None, migration_out: None }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.387436176Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-14T04:58:09.345955905Z }, migration_in: None, migration_out: None }","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.390478484Z","hostname":"sock","pid":1572,"instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","propolis_id":"2c73ff36-1861-4bac-ad42-72e89f1dbf2d","instance_id":"f872674e-3abd-4856-abe7-9936da747fcf","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.960700396Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:09.96083758Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bcef656b-3c66-479b-acb0-df8a08257172","remote_addr":"[fd00:1122:3344:101::c]:64409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.064102881Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.064164842Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.064172243Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2fd10b61-ce2c-4db1-bbe8-977521bd63e9","remote_addr":"[fd00:1122:3344:101::c]:64409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.079742783Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.079997731Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4116cbf1-f3fd-48e1-b3be-36f885215b00","remote_addr":"[fd00:1122:3344:101::c]:36532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:10.087952061Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1a9c14d8-2319-4737-ab31-0e718244903d","remote_addr":"[fd00:1122:3344:101::c]:64409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.676105652Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.676317188Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6739647f-7ac0-405d-a4f3-528a3d82adac","remote_addr":"[fd00:1122:3344:101::a]:48884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.69988024Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.699982914Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2266fe79-aa1c-4901-994b-b228c95de4d9","remote_addr":"[fd00:1122:3344:101::a]:39063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.739622201Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.739642472Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b8cb0da2-e3f8-4585-95ab-79604e7dff1e","remote_addr":"[fd00:1122:3344:101::a]:39063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:11.747905751Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"57646b8d-7de4-45fb-93ab-1a4471ca9112","remote_addr":"[fd00:1122:3344:101::a]:39063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:12.545133848Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57555"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:12.545224951Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a50de7f4-d4b4-4cf7-8c43-42d0af539886","remote_addr":"[fd00:1122:3344:101::a]:57555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:12.59699053Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:12.597058922Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5ed4a383-060b-4d25-94c8-ee79b93f0aa6","remote_addr":"[fd00:1122:3344:101::a]:57555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:12.608697539Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"28ea0855-1441-4ed5-b335-b9f7365d2b4b","remote_addr":"[fd00:1122:3344:101::a]:57555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":271,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.354581256Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.709298919Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.71002989Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.743834229Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.753635313Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:14.889768424Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.00874866Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.010961234Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.011662884Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.012333484Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.059149909Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.059770977Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.062002452Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.075828762Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.076350617Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.079027695Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.107284483Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-14T04:58:15.107821949Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.61202311Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.612146144Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a682bc3f-f58b-4ac6-a73b-effc05793c54","remote_addr":"[fd00:1122:3344:101::c]:64953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.686678662Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.686747894Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8afb27cb-f129-48d1-9686-d52ebaadf9ee","remote_addr":"[fd00:1122:3344:101::c]:64953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.703058786Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0ee10993-404c-473b-a855-ba885982758e","remote_addr":"[fd00:1122:3344:101::c]:64953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.718666398Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.718765471Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bc10c70b-b99e-4a79-b0e0-36582f00bc74","remote_addr":"[fd00:1122:3344:101::b]:42722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.757260746Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41609"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.75738871Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"67192452-d2c6-4080-8c5b-e428d8e5940c","remote_addr":"[fd00:1122:3344:101::a]:41609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.765723221Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.765741772Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a5687675-c71e-4b05-852c-d4c056810890","remote_addr":"[fd00:1122:3344:101::b]:42722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.785417671Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"64be32cf-67bb-498e-8fc1-675f3b9ccbb5","remote_addr":"[fd00:1122:3344:101::b]:42722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.804893115Z","hostname":"sock","pid":1572,"sled_id":"c073d51c-abaa-48df-abb6-2f053e9c1bec","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.804927926Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bf163ca5-48b3-4d75-a550-280618f0b932","remote_addr":"[fd00:1122:3344:101::a]:41609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.80576515Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41993"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.805853603Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.851452493Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.915796466Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e7720ef1-1b49-4a34-82d3-7f474544a731","remote_addr":"[fd00:1122:3344:101::b]:41993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109974,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916040673Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42387"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916113816Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55791"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916163107Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54314"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916322352Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6f88cd9a-4cb2-4184-ad5d-d266dffe0649","remote_addr":"[fd00:1122:3344:101::a]:41609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":343,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916350113Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b3f4722f-9192-485f-9b3d-ddab68216741","remote_addr":"[fd00:1122:3344:101::b]:41993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":278,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.916541228Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.91661268Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3743af40-bde9-4f0d-aa4a-31228a4436e5","remote_addr":"[fd00:1122:3344:101::b]:55791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":253,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.923382856Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6b3ed219-4c3e-4451-bb6d-01b21ac5a7b9","remote_addr":"[fd00:1122:3344:101::c]:42387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6789,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.938849174Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47854"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.938940887Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.93939502Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"23a17801-acd3-4132-a4ba-baf85ba88318","remote_addr":"[fd00:1122:3344:101::a]:40170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":257,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:15.962417836Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.027003127Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.027073049Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"092bb80b-0a0f-4eb5-9411-8d2ac6ea9756","remote_addr":"[fd00:1122:3344:101::c]:54314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110656,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.031662182Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0a99b192-f261-43f1-8939-5477a54632d7","remote_addr":"[fd00:1122:3344:101::c]:54314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":307,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.064279496Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.12655114Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"288b57bc-3f59-4d5a-aafa-d867d0756d27","remote_addr":"[fd00:1122:3344:101::a]:47854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187405,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-14T04:58:16.12691318Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"891d41fe-5a93-40e5-943a-b2d3b5953dfc","remote_addr":"[fd00:1122:3344:101::a]:47854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"}