[ Nov 6 00:14:31 Enabled. ] [ Nov 6 00:14:31 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Nov 6 00:14:31 Method "start" exited with status 0. ] [ Nov 6 00:14:31 Rereading configuration. ] [ Nov 6 00:14:31 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.576541844Z","hostname":"sock","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.576688708Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.62861134Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.628657801Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.741243594Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.750102857Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:31.757780185Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.362428796Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503719961Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503744932Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503833544Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503886516Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503894066Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.503911487Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.507141943Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:32.507147853Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.507157533Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.507167233Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.507181744Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.507290767Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.50737195Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.50739123Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.50740139Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.518535541Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.518552962Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:32.617257932Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.093449339Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.09347882Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.09348339Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.093493191Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.093502681Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.093506961Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.093516301Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.102174138Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.102189949Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.192556502Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.66271734Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662762532Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662798933Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662803993Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662813043Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662819923Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662825884Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662856775Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662863695Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662876575Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662880905Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662889246Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662892466Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662897096Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662946447Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662953647Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662970788Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662975138Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662978498Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662982199Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.662997049Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663005159Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.66303067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.66303548Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663077251Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663097822Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663102732Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663124603Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663128843Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663132953Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663136773Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663143143Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663149313Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663152904Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663172874Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663177554Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663180844Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663185465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663190114Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663213465Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663218146Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663221635Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663224906Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663231406Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663250486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663270117Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663273937Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663277207Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663281237Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663285328Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663288488Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663308908Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663312878Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663316888Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663324129Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663327389Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663348539Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663352249Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.66336084Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.66336556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663410241Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663473753Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663492434Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663502874Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663515384Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663522615Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663527765Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663561256Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.663565366Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.674856121Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:733"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.674887452Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999919097s","next_cleanup":"Instant { tv_sec: 1336, tv_nsec: 650738168 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.674929543Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.674966204Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.675021456Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.677394976Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.6778429Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.679821518Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680086006Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680095996Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680155648Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.68020583Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.68022101Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.68022778Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680251611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680285982Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680292622Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680301263Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680305353Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680309723Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680342424Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680375935Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680695774Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 3a58cbfb-5015-41c2-9142-616f1a1558aa,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 7c647f25-eb29-494d-840d-11658133af45 (sled),\n rack_id: 3a58cbfb-5015-41c2-9142-616f1a1558aa,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680938141Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680942962Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680969733Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.680981563Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.681136507Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.681260201Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.681315263Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 7c647f25-eb29-494d-840d-11658133af45 (sled), rack_id: 3a58cbfb-5015-41c2-9142-616f1a1558aa, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.681760056Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4206"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.681826628Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4588"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.686121685Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.691977529Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.692085383Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696238116Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696300248Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.6963964Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:64091","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 7c647f25-eb29-494d-840d-11658133af45 (sled), rack_id: 3a58cbfb-5015-41c2-9142-616f1a1558aa, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696519234Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696535224Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696562855Z","hostname":"sock","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696575176Z","hostname":"sock","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696580266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696585126Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696601906Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696620507Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696635318Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696652918Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696659438Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696663608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.696697989Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.69670592Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.712980853Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.733191133Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.750211648Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.759437452Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:408"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.759481353Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.759486073Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:420"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.759548955Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 66.974302ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.76273379Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0565235e-2fee-4b2b-91cd-21f8dfb5dcb1\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0565235e-2fee-4b2b-91cd-21f8dfb5dcb1\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_0a098d83-7312-4c1d-895e-4802f8c14d04/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.77284805Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_0a098d83-7312-4c1d-895e-4802f8c14d04/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.796031458Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_0a098d83-7312-4c1d-895e-4802f8c14d04/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.79607556Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.802509921Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:435"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.802517321Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_0a098d83-7312-4c1d-895e-4802f8c14d04/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.802524951Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.831293416Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 116.465493ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.832638975Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b3e3c7a3-503d-4794-ab28-cd728ec21de1\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b3e3c7a3-503d-4794-ab28-cd728ec21de1\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.886172705Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.950539366Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 109.658365ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:33.951168435Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9b19943-5294-4652-b8d1-df13d1da6acf\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9b19943-5294-4652-b8d1-df13d1da6acf\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.062744227Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 138.192419ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.063251882Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7349258-a36a-4b1d-811b-cffb67b3abec\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d7349258-a36a-4b1d-811b-cffb67b3abec\" }"} {"msg":"Mounting oxi_0a098d83-7312-4c1d-895e-4802f8c14d04/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.106158126Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.117949156Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.118037758Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.118065029Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.118077739Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.202318671Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 298.726922ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.202949169Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6fbf52a6-8f34-4a25-bd65-dd32f65f9adb\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6fbf52a6-8f34-4a25-bd65-dd32f65f9adb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.501934306Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 201.78836ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.50275101Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5187c74f-cc52-4f93-9f31-4ec8e29dd4be\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5187c74f-cc52-4f93-9f31-4ec8e29dd4be\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.704806579Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 416.495437ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.705401537Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b928dd3-846d-47de-93a1-60572ad1f165\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:14:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b928dd3-846d-47de-93a1-60572ad1f165\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.842070084Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.847755363Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:58224","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.847789164Z","hostname":"sock","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.847850116Z","hostname":"sock","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:58224","producer_id":"7c647f25-eb29-494d-840d-11658133af45"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:34.849147155Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"190.62251ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.850032561Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.850037821Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.850105193Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:917","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:34.850110683Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:35.040765893Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"300.391468ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:35.123735256Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:35.343416539Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.449899379s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:36.795399067Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.233922806s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275846181Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2664"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275880692Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2882"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275886002Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2534"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275890142Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2581"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275901092Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2963"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.275908663Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.277878631Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2807"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.279506209Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2575"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.279624933Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:37.279791618Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 73.642205ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:37.641511187Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 58.336469ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:37.85210909Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 173.418768ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:38.035651118Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 499.140297ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:38.375622262Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.007898409s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:38.983140898Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:39.031305238Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.311691249s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 581.568375ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:40.102667086Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 511.985854ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:40.759115245Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.480258247s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:41.431452176Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:43.344769151Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.378682028Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.379197253Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380085729Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.38011139Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380125211Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380139641Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380294605Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.38046301Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380518112Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380530332Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.380547143Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:64091","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.625844976Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.625932498Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.625943059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.6259945Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.626001891Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.626018891Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.626035532Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.626049252Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.626067662Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.630884016Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4230","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.630898166Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4246"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.630906666Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.940877159Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62852"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.941135907Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.941147137Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.941190138Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.941208309Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.941310592Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"a7879461-9151-46eb-b791-ed1cb7883fe8","remote_addr":"[fd00:1122:3344:101::1]:62852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":291,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.971032254Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:33120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.971184868Z","hostname":"sock","pid":1572,"uri":"/sled-role","method":"GET","req_id":"7229cfcb-54b0-40bb-8f0e-98a4b4d734b8","remote_addr":"[fd00:1122:3344:101::1]:33120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.971793746Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.971825127Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.972210129Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:43.97257196Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:886"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: c8d72ed0-d8e8-4f66-b079-0c6e71be02de, pool_id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: aad1632d-f381-4492-ae3e-ca57ba56ff94, pool_id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: f87dcbd1-c137-44e3-baf2-3b70d9a0ebb8, pool_id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 32048f19-6860-424a-8eff-4f18fc896448, pool_id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 0ba9ff81-5279-4613-8430-fe63252470c5, pool_id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 1a8ba944-15ac-4cd9-b128-cf368a9c0ced, pool_id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: f241f3e3-afef-480e-8a10-ca7c5f9b898b, pool_id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: e29698d0-e020-4087-9475-9a122b08a7cf, pool_id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c55604b6-8a1c-4f82-97b2-aff60c0d9718, pool_id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001613232Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:376"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001686434Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48626"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001922811Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: c8d72ed0-d8e8-4f66-b079-0c6e71be02de, pool_id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: aad1632d-f381-4492-ae3e-ca57ba56ff94, pool_id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: f87dcbd1-c137-44e3-baf2-3b70d9a0ebb8, pool_id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 32048f19-6860-424a-8eff-4f18fc896448, pool_id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 0ba9ff81-5279-4613-8430-fe63252470c5, pool_id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 1a8ba944-15ac-4cd9-b128-cf368a9c0ced, pool_id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: f241f3e3-afef-480e-8a10-ca7c5f9b898b, pool_id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: e29698d0-e020-4087-9475-9a122b08a7cf, pool_id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c55604b6-8a1c-4f82-97b2-aff60c0d9718, pool_id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001943432Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001953732Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001962992Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001967892Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001982953Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001987643Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001992973Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.001997273Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.14168353Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.141730522Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:44.345681007Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.051253956s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:44.592851645Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.82069286Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:44.852491404Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17933885404809741736"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:44.852585896Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13133400431465011935"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.867748307Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875446395Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875496727Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875504917Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875512127Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875554218Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875670672Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.875676832Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.911941739Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f049eab6-1590-47c8-b087-bc798a2cfc73/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:44.91975065Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_f049eab6-1590-47c8-b087-bc798a2cfc73","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.093789677Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.101593779Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.10163083Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.101677122Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.101687712Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.101694562Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.112131472Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.112155802Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.206284557Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.213920364Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.213944945Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.213950995Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.213982186Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.214031697Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.214167001Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.214173871Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.248218942Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9a4fd05b-1fb2-49d1-ab40-b32093e6fd86/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.256475717Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_9a4fd05b-1fb2-49d1-ab40-b32093e6fd86","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.427561907Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.435140662Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.435171493Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.435177113Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.435182043Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.435186253Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.444562382Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.444585062Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.536400578Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.543996614Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.544026044Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.544032155Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.544062786Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.544072756Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.54420842Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.54421701Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.576999054Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.585068793Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:45.59572729Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_25d63394-e252-4651-bd2a-cb7cb7ea5b3e","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.755954427Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.764519381Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.764557972Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.764580373Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.764587063Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.764591313Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.775063684Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.775091115Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.871073624Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878541076Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878572887Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878581987Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878593088Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878601038Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878718621Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.878724872Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.91203441Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_aaf2d272-1feb-4e24-ba54-7075c6ab94d1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:45.918981167Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_aaf2d272-1feb-4e24-ba54-7075c6ab94d1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.093145507Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.101215587Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.101245098Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.101250838Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.101257868Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.101262178Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.110984447Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.111006008Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.209255705Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218181759Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.21821254Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218231531Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218305673Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218316523Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218472958Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.218489559Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.254441446Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b89e85b3-c4ff-4874-bdbd-5e8cca783de4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.262525366Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_b89e85b3-c4ff-4874-bdbd-5e8cca783de4","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.433870723Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.441653684Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.441698606Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.441739247Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.441746657Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.441751847Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.451309721Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.451337912Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.543981862Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551503975Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551532276Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551538287Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551583478Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551591908Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551759263Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.551765433Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.58534324Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a62f7ed5-f062-4f65-8692-31c405e42f70/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.592785871Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:46.597780749Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_a62f7ed5-f062-4f65-8692-31c405e42f70","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.760118889Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.768309842Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.768344133Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.768378054Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.768384565Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.768390555Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.777949269Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.777969499Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.886858832Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894356745Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894383555Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894405486Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894442677Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894462348Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894573441Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.894580071Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.929075935Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_392a3437-26be-4bcb-8f19-18fd4492b7e0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:46.936545457Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_392a3437-26be-4bcb-8f19-18fd4492b7e0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.117120538Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.124761675Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.124801276Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.124807196Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.124813836Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.124818147Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.134188905Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.134215926Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"60f04be8-23a0-4089-b45c-357c38441897 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.231136803Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239270694Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239300335Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239307906Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239350867Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239361757Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239482721Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.239489451Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.274984965Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_60f04be8-23a0-4089-b45c-357c38441897/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.282251711Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_60f04be8-23a0-4089-b45c-357c38441897","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.447628891Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.455185655Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.455226246Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.455246257Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.455255017Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.455261127Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.464928314Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.464958775Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0ff26524-4a22-4458-bf83-380365ee3e3b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.572881649Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.5806439Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580685321Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580713412Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580761723Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580770624Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580923268Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.580929908Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:47.600506639Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.618776172Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0ff26524-4a22-4458-bf83-380365ee3e3b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.626233613Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_0ff26524-4a22-4458-bf83-380365ee3e3b","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.801351202Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.808976079Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.80900885Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809067491Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809199455Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a098d83-7312-4c1d-895e-4802f8c14d04 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a12a67dd-438b-47b6-9a37-99f4c825ac9d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809259387Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809268957Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809399411Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809642559Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:890"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809659979Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809665529Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:916","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809670009Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:921"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809674909Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:926"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.80968828Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.8096943Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:932"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809737131Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809747422Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.809823954Z","hostname":"sock","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"76a3ebae-9d12-4169-a03e-678b5a28ba18","remote_addr":"[fd00:1122:3344:101::1]:48626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3808022,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.840812194Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.840907157Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50769"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.841168615Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.841175335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.893737685Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.893783797Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.945580244Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.945616496Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.994872127Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.994914619Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.994920469Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:47.99495246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.036547515Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.036581026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.036587966Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.089568109Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.093348901Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.097079682Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.109635035Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.113899511Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:48.117761926Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/7c647f25-eb29-494d-840d-11658133af45)","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:48.602469717Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:49.678904065Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:49.678962477Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:49.679167473Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.162056619Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.639762032Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.639874505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.639972068Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 34.16487ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.647957105Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.648006347Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:50.651958734Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 67.116778ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.691430826Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 160.020672ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.767893866Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 29.126858ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.774160932Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 58.878212ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.780526461Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 144.869646ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.810392008Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 144.378941ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.897857375Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 439.310833ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:50.971917893Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 180.056035ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.016055123Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 284.662762ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.06984078Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 427.945309ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.214956849Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 369.633605ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.363761367Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 726.913202ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.421221333Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 672.515598ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.65251233Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 1.086066386s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:51.743072088Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 671.565887ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:52.158008757Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 1.155865839s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:52.335161177Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 1.441842322s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:52.838312835Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 1.056294868s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:52.845454907Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:53.345890524Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2686974636765756785"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:53.346004657Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6562736876880607994"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 712.388831ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:53.499532435Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.709580962Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4254"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.709628283Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4264","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.715628861Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.810461647Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4391"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.810497158Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4418"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.84829773Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4431"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.848327021Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4295"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.898090768Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4337"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.8981257Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4398","rack_subnet":"fd00:1122:3344::"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 903.536438ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:53.911341822Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.918941467Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4409"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.918951568Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4451"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.986631817Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4483"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:53.986667338Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.087487341Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4530"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.087537913Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.088389328Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:40680"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.088558283Z","hostname":"sock","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"972fed65-0178-413e-b876-3b29bfc67ef6","remote_addr":"[fd00:1122:3344:101::2]:40680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.120039418Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.152172132Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.152280515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.152668406Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.152687297Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.188793079Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 627.880596ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:54.22117079Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236882637Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236938358Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236946788Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236953859Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236959869Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.236974569Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23698016Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23698543Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23699172Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23699805Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23700362Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23700984Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.23701597Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237021081Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237027161Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237033191Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237037331Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237041221Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237053612Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237057842Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237061792Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237065752Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:54.237069712Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388\") failed: Property not found. retry in 1.188906302s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:54.289289622Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d\") failed: Property not found. retry in 720.731029ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:54.824476042Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6\") failed: Property not found. retry in 748.207787ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:54.85808406Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:54.859287215Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17933885404809741736"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:14:54.859410349Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13133400431465011935"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.628793621Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_51df9631-cef5-49ae-8cfd-f0c05bdac388"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.628843583Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_7102227a-61f8-4dda-a9a0-f6678fae71c6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.628861933Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_ad2fb963-23a1-4187-b186-e731e9dae98d"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.628872153Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.629148742Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.62942215Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3ba5e178-2d04-4c20-ad5a-176efb717c0d","remote_addr":"[fd00:1122:3344:101::1]:50769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7788416,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:55.631959545Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"failed to write DNS configuration (will retry in 131.796664ms)","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:56.012265516Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:596","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.182258363Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.235560505Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.279932353Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.279942533Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.311713466Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.312547191Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.312554462Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.342189271Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.342274804Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60997"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.342466239Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.3424735Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.362442803Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.362472164Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.380296733Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.380328414Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.397073611Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.397122062Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.397301898Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.397428301Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.408348235Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.442042006Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.443018205Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.443023205Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.474614623Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 138, 94]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.474633434Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.475074027Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.475090867Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.475458978Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.47584923Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.498462261Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:56.517361212Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:57.125302531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:14:57.125419774Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 59.280046ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:57.266347859Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 81.323061ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:57.346331673Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 239.754921ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:57.461826512Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 592.954525ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:57.713543575Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 1.093588317s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:58.315158387Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 1.376405377s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:14:59.418182054Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 651.636198ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:00.804943206Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe\") failed: Property not found. retry in 1.48866489s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:01.465973451Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:02.974200889Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:02.974259971Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:02.974641662Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:02.975008243Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8b2078a6-57ab-45db-938a-9cc53783eefe","remote_addr":"[fd00:1122:3344:101::1]:60997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6632667,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:02.976956301Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:653"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:03.006633882Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:03.006770196Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:59674"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:03.015488655Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:03.015640679Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"b00ad3f4-3c45-4e60-b99f-3a3b2baa13b9","remote_addr":"[fd00:1122:3344:101::1]:59674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8735,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:03.01869722Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:03.194395416Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:03.19450921Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41029"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:03.348937545Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2686974636765756785"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:03.349068849Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6562736876880607994"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:04.861024077Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17933885404809741736"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:04.861296855Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13133400431465011935"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:05.502681617Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"478319522166758437"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:05.572867941Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11719336887255222848"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:05.631138491Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1947079445530543555"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_cdbd7d61-5fa7-425e-9630-03453e0bcbbe': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-06T00:15:07.201958857Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.20207345Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"430fedaf-7b3f-4db5-aeb3-bfc3329de255","remote_addr":"[fd00:1122:3344:101::1]:41029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007510,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:07.204922395Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.329087531Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.329228105Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.337214553Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"d3c82af7-cd0d-43a1-95d5-c2f06a7eb6a1","remote_addr":"[fd00:1122:3344:101::1]:35677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7911,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.340373707Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:07.340392237Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.582262728Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.582384281Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.589940406Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"f088c060-4f0b-49f6-a1bf-a92de2bcb628","remote_addr":"[fd00:1122:3344:101::1]:36480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7500,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.593061829Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:07.593066929Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.787710157Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.787820371Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.79520734Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"0bd7c99b-df9e-4fa9-9bdc-fb29bbb21708","remote_addr":"[fd00:1122:3344:101::1]:62995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7334,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:07.796839239Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:07.796845169Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.133828763Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.133932126Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.141284614Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"01b11ec4-b1a8-4ee2-b3c9-48cff1e8c637","remote_addr":"[fd00:1122:3344:101::1]:50532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7211,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.143171361Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.143180501Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.428249294Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.428381028Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.437006064Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"5fe23bac-6e34-4d8f-adaf-cd946af8bd58","remote_addr":"[fd00:1122:3344:101::1]:53486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8563,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.438457237Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.438462467Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.043023498Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.043184993Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.064184426Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"85d7d49a-11db-4de0-a680-44e11e38c202","remote_addr":"[fd00:1122:3344:101::1]:48217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20903,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 51180378, ip_addr: 3.12.243.90, stratum: 5, ref_time: 1730852107.9879916, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.067976098Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.067983109Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.098021461Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.098102563Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60254"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.09832377Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.09833284Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.11755206Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.117594502Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.135802752Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.135836013Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.154901279Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.154963291Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.208294194Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.208341456Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.261556576Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.261612537Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.317413544Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.317459175Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.367506291Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.367543682Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.41733237Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.417368762Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.417581218Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.417726422Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449279629Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449343491Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449354531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449362012Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449367452Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.449432954Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"95469182-1458-4e0c-bfdb-4dcb51b12969","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":351258,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.449592468Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"121.166351ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"95469182-1458-4e0c-bfdb-4dcb51b12969\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"95469182-1458-4e0c-bfdb-4dcb51b12969\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.571756465Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.572145337Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.572158627Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.589564374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.589603405Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.607252759Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.60728657Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.623765819Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.6238049Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.642218167Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.642277589Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.660563982Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.660609013Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.677764482Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.677809134Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.695729146Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.695771787Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.712873715Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.712923106Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.713135883Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.713286757Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.743734421Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.743774802Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.743782102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.743790753Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.743797113Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.743898956Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9b11e1b0-1278-4baa-8129-93279489890d","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized)]","latency_us":171955,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:08.74404056Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"129.141353ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b11e1b0-1278-4baa-8129-93279489890d\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b11e1b0-1278-4baa-8129-93279489890d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.87438377Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.874557965Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:08.874566125Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.011293115Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.011338866Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.029385792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.029427263Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.046374166Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.046411847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.063955698Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.06399446Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.081053766Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.081097807Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.100126532Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.100165743Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.118423285Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.118462066Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.136263205Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.136311067Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.136529593Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.136639596Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.167009848Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.167053879Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.167061909Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.16707138Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.16709085Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.167178133Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b5cb392a-d8f1-4933-9749-bcbd03e252d2","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":173713,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.167319987Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"162.774915ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5cb392a-d8f1-4933-9749-bcbd03e252d2\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5cb392a-d8f1-4933-9749-bcbd03e252d2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.331723058Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.332012286Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.332024026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.349875847Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.349919718Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.367140799Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.367185321Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.385231726Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.385280888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.402984393Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.403024354Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.420430171Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.420463802Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.437480397Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.437513369Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.454344308Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.45439041Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.471362974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.471410535Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.47160301Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.471725944Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503490067Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503526428Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503533329Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503540629Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503547129Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.503672063Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ed5044a2-5a4f-417c-ace5-d4f80a4a70c1","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":171818,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.503832247Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"216.187525ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ed5044a2-5a4f-417c-ace5-d4f80a4a70c1\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ed5044a2-5a4f-417c-ace5-d4f80a4a70c1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.720936583Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.721099618Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.721111668Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.740315439Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.74036824Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.759341883Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.759372434Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.777189413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.777235044Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.795439945Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.795468836Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.81345824Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.813480411Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.831257879Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.8313025Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.849607023Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.849637524Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.867126993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.867165545Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.867406992Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.867552206Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.898983269Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.89901348Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.89902202Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.899028391Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.899035131Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:09.899164364Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"80a07630-dc37-4d14-a80a-e6cb6c2849dd","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized)]","latency_us":178144,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:09.899320389Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"117.905064ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"80a07630-dc37-4d14-a80a-e6cb6c2849dd\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"80a07630-dc37-4d14-a80a-e6cb6c2849dd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.018731565Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.019028033Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.019037294Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.038094859Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.038145731Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.055576658Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.055610739Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.072442409Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.07249073Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.092112503Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.092153604Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.109379596Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:10.109413597Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.005298728Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.005332138Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.024199589Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.02423903Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.042211033Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.042254225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.042452971Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.042563804Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.073934875Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.073971176Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.073976576Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.073982737Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.073987547Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.07410117Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"10b02539-29c8-4091-b204-2514ab7c9d60","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized)]","latency_us":178236,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.074275115Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"245.216026ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10b02539-29c8-4091-b204-2514ab7c9d60\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"10b02539-29c8-4091-b204-2514ab7c9d60\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.320695711Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.321036151Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.321053972Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.34119709Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.341229061Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.360806172Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.360844893Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.3792649Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.379291831Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.397076449Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.39710152Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.415338671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.415367712Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.432962054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.432990555Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.450392332Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.450419183Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.467557841Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.467583592Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.467775128Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.467900442Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.49815436Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.498199411Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.498206171Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.498212962Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.498235412Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.498354646Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7422510c-2578-4815-a9f1-e4c667b314a4","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized)]","latency_us":177504,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.49850562Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"205.052892ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7422510c-2578-4815-a9f1-e4c667b314a4\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7422510c-2578-4815-a9f1-e4c667b314a4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.703902348Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.704263999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.704272689Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.724487689Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.724526671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.742633118Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.74268006Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.760400696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.760438777Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.780516603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.780550214Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.79926016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.799303211Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.818346416Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.818381487Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.836783334Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.836829675Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.855579381Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.855628643Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.855812548Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.855941392Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.887807978Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.887837439Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.88784389Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.88784897Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.88785504Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:11.887963263Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7ca882cc-2521-45c1-b7db-e4057d8c2b6e","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized)]","latency_us":183889,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:11.888127228Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"354.871384ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ca882cc-2521-45c1-b7db-e4057d8c2b6e\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7ca882cc-2521-45c1-b7db-e4057d8c2b6e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.243898409Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.244276601Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.244293981Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.263935026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.263954866Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.283450937Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.283480428Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.301769252Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.301804103Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.32018322Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.320205651Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.338736493Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.338756623Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.35677489Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.35679487Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.374989082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.375009893Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.392794622Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.392828593Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.393001918Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.393144472Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426407403Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426454834Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426462904Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426468984Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426474525Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.426556807Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fb597085-fbad-401e-8cfe-8e8cbc6bf6c3","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized)]","latency_us":182953,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:12.426690731Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"526.452614ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb597085-fbad-401e-8cfe-8e8cbc6bf6c3\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb597085-fbad-401e-8cfe-8e8cbc6bf6c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.952428385Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.952782833Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.952791683Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.97183008Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.971871641Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.989469645Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:12.989500526Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.006644426Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.006682148Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.025137247Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.025171768Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.043538015Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.043582426Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.060801439Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.060834089Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.079472315Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.079501215Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.09680934Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.096857812Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.097089869Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.097213643Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129200615Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129250806Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129259916Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129266446Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129274977Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.12936444Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"750c5cb3-fada-42d4-afd4-26c96eb8639c","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized)]","latency_us":177251,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.129484383Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"485.309781ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"750c5cb3-fada-42d4-afd4-26c96eb8639c\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"750c5cb3-fada-42d4-afd4-26c96eb8639c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.614971956Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.615123241Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.615129971Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.633827027Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.633869059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.650692639Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.65073175Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.667502373Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.667539901Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.68498061Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.685010041Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.703547293Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.703581204Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.721130846Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.721163567Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.74008481Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.740123742Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.757705365Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.757736006Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.757951192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.758054105Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789220953Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789260274Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789267375Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789274195Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789282705Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:13.789411529Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"05d3e25a-07c4-492d-9aca-ba9a965c9048","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":174844,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:13.789571874Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"380.564228ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05d3e25a-07c4-492d-9aca-ba9a965c9048\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05d3e25a-07c4-492d-9aca-ba9a965c9048\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.170345299Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.170659688Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.170681429Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.189740136Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.189788918Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.207182756Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.207232027Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.224090629Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.22413046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.242619111Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.242669772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.26072714Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.260768331Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.278221401Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.278261662Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.296874116Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.296915817Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.3157574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.31581398Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.316057697Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.316208792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.348712509Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.348758601Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.348768451Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.348776301Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.348785931Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:14.348900345Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b1a9a785-4cc9-41c9-8e8b-c167feb6a966","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized)]","latency_us":178893,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:14.34907737Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.111378389s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1a9a785-4cc9-41c9-8e8b-c167feb6a966\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1a9a785-4cc9-41c9-8e8b-c167feb6a966\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.459098675Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.459443135Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.459451266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.479528953Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.479571828Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.498200369Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.49825179Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.5160268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.516057511Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.534904132Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.534938123Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.552497616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.552535787Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.570516372Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.570563983Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.575211932Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"305.613907ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.589322412Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.589359663Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.606777421Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.606813753Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.607001198Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.607118011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640488525Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640520296Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640527286Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640532526Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640536336Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:15.640592378Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fb8ecc45-20b5-400b-bf91-88b225256dc2","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":181839,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:15.640723292Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"782.073867ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb8ecc45-20b5-400b-bf91-88b225256dc2\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb8ecc45-20b5-400b-bf91-88b225256dc2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.421006265Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.421292929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.42130268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.441073398Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.441103699Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.458370253Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.458397954Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.476783761Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.476820542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.494694474Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.494731295Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.513463413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.513489484Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.532173Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.532200791Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.552247268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.552275808Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.56978892Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.569828871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.570052728Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.570188982Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.601927627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.601983688Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.601995689Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.602003589Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.602011319Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:16.602106702Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fae6bf1c-8f03-4c19-8b46-0c7b9bd71040","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":181471,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:16.602257016Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.346790967s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fae6bf1c-8f03-4c19-8b46-0c7b9bd71040\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fae6bf1c-8f03-4c19-8b46-0c7b9bd71040\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.945845928Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.946191948Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.946203118Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.967251165Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.967294566Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.983827149Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:17.9838583Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.003048741Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.003085092Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.022714047Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.022749888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.041783275Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.041818666Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.061499492Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.061539603Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.078648683Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.078683084Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.09467279Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.094707431Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.094928288Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.095041251Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125008873Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125043394Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125050114Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125068015Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125073295Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:18.125168698Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"87b37cee-723e-46d4-b71e-ec38baa7d789","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":179733,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:18.125331313Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.101731723s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87b37cee-723e-46d4-b71e-ec38baa7d789\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87b37cee-723e-46d4-b71e-ec38baa7d789\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.22497763Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.225300869Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.22531846Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.245253003Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.245288374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.263757574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.263797305Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.281209324Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.281255985Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.300242711Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.300287692Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.318052051Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.318085672Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.336343575Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.336379276Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.355508196Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.355546947Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.37377418Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.373831322Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.374001316Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.374155191Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405594747Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405627268Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405633568Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405638818Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405644669Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:19.405742672Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8e4c5434-c8be-476d-8207-9d9d3ef6fe9b","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":181160,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:19.405884326Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.365089404s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e4c5434-c8be-476d-8207-9d9d3ef6fe9b\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e4c5434-c8be-476d-8207-9d9d3ef6fe9b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.768637999Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.768973039Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.76900955Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.790434308Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.790466569Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.809910798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.809940359Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.828561563Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.828613155Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.847451705Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.847486976Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.865611766Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.865639067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.883478778Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.883503969Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.90135429Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.901380921Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.919694396Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.919729569Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.919929273Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.920078538Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952287627Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952332628Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952342598Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952351628Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952360689Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:20.952460082Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"aad399c5-aef0-45ce-bfe2-e364fc682d99","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized)]","latency_us":184215,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:20.952617396Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.148086678s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aad399c5-aef0-45ce-bfe2-e364fc682d99\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aad399c5-aef0-45ce-bfe2-e364fc682d99\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.098854882Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.099261984Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.099275624Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.118503417Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.118539738Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.136824832Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.136866164Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.155801367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.155842068Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.173605717Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.173649608Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.190713367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.190752638Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.207706802Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.207747104Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.226574034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.226621045Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.244214289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.244256101Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.244476487Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.244588651Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276335316Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276382767Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276393027Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276400958Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276410298Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:22.276519111Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fbba35b3-1691-428f-b822-818025315e05","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":177966,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:22.276712607Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.217407813s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fbba35b3-1691-428f-b822-818025315e05\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fbba35b3-1691-428f-b822-818025315e05\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.491860843Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.492200063Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.492207293Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.511869469Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.51192343Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.52938634Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.529421241Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.546258543Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.546301004Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.564739993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.564780664Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.580231114Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.580277305Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.598116526Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.598149867Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.615527665Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.615564956Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.633996335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.634052106Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.634239622Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.634376216Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.665711239Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.66574505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.66575369Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.6657601Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.66577429Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:23.665883624Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3d1c37d2-3ded-4ddc-9794-98573b42b159","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized)]","latency_us":174337,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:23.666042178Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"2.175155421s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d1c37d2-3ded-4ddc-9794-98573b42b159\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3d1c37d2-3ded-4ddc-9794-98573b42b159\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:24.031635203Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:24.031929511Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"6b6fc335-9dd9-44a0-ab0a-a6894d78cdf5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43537","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.836973239Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.837312309Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.837323019Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.858341005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.858367135Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.876784584Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.876808324Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.894219993Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.894240183Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.91227876Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.912299781Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.928758331Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.928777812Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.945745187Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.945765427Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.963345281Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.963373882Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.981936704Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.981980845Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.982202952Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:25.982343916Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.012771082Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.012819974Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.012828744Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.012834984Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.012852204Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:26.012951927Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b0dc83a2-f15a-47bb-a3d4-5f48f2a7c3a8","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized)]","latency_us":176297,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:26.013137563Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"2.040535539s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b0dc83a2-f15a-47bb-a3d4-5f48f2a7c3a8\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b0dc83a2-f15a-47bb-a3d4-5f48f2a7c3a8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.049793415Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.050176606Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.050190127Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.070640265Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.070691177Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.090212178Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.090266609Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.108209374Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.108255165Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.12656974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.126609562Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.144440092Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.144477933Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.163639504Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.163691745Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.181073313Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.181115894Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.198362707Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.198414159Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.198647136Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.19877413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.231308538Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.231338279Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.231344529Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.23134968Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.23135452Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:28.231459913Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d837257e-95d2-4aeb-b732-ea1716e6e3ba","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized)]","latency_us":182012,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:28.231619355Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"3.07659665s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d837257e-95d2-4aeb-b732-ea1716e6e3ba\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d837257e-95d2-4aeb-b732-ea1716e6e3ba\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:30.873541187Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"558.022884ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.300624101Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.30090691Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.30091491Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.321925656Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.321977627Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.340250971Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.340289452Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.358201386Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.358235367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.375120949Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.375167211Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.393679212Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.393722663Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.413127651Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.413169952Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.431323792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.431385324Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.450246596Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.450285007Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.450513324Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.450655288Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483565798Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483603299Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483609599Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483614729Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483619619Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:31.483681611Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fc635dff-4787-425c-8b83-74bf0a81f4b2","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":183437,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:31.483845616Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"3.883006001s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc635dff-4787-425c-8b83-74bf0a81f4b2\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc635dff-4787-425c-8b83-74bf0a81f4b2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.357265967Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.357539815Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.357549476Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.379486929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.379512089Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.399527405Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.399549716Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.418559772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.418583032Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.436375452Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.436397873Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.454538083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.454568514Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.472416355Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.472436906Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.490835143Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.490860793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.508167139Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.50819331Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.508399446Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.50853323Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.541127561Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.541164312Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.541171372Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.541179592Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.541185662Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:35.541282355Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"61f13b0b-f094-4526-b99a-3d0352d0a46a","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized)]","latency_us":184396,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:35.54143695Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"5.663638768s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61f13b0b-f094-4526-b99a-3d0352d0a46a\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61f13b0b-f094-4526-b99a-3d0352d0a46a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.190561594Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.190924335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.190954886Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.212484767Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.212509077Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.232026428Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.232056979Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.249812158Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.249834868Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.267519005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.267541266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.286455419Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.28650394Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.305105394Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.305150275Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.323003177Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.323047438Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.3402352Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.340302082Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.340537299Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.340661292Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.370888402Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.370927403Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.370935984Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.370958944Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.370967285Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:41.371058647Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"12684009-73aa-4dfd-ac70-157690e40b68","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized)]","latency_us":180839,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:41.371218622Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"3.364146138s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12684009-73aa-4dfd-ac70-157690e40b68\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12684009-73aa-4dfd-ac70-157690e40b68\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.726655395Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.726951472Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.726963942Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.746871755Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.746914736Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.767516849Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.767551793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.786962828Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.78700204Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.805218662Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.805251593Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.824784074Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.824817915Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.844066159Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.84411711Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.863676292Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.863711943Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.883714059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.88376293Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.883978287Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.88410025Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916128314Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916176935Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916187006Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916193786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916201746Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:44.916315449Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ff7522b1-3526-454d-831d-c07ab7d68cc3","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":190057,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:44.916490184Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"8.13666204s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff7522b1-3526-454d-831d-c07ab7d68cc3\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ff7522b1-3526-454d-831d-c07ab7d68cc3\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:46.450828961Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"1.069334274s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.032129866Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.032332092Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.032337592Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.054834542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.054864213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.075058094Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.075105835Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.094611616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.094636047Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.113133697Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.113155878Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.131564186Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.131586067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.149652335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.149674005Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.168207507Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.168230648Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.185969556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.186018017Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.186213683Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.186358877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.219332179Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.21936785Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.21937321Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.21938047Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.219389301Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.219536153Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d51b801b-dad5-40ae-a83d-408eae20c518","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized)]","latency_us":187807,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:15:53.21969618Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"11.72586651s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d51b801b-dad5-40ae-a83d-408eae20c518\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:15:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d51b801b-dad5-40ae-a83d-408eae20c518\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.984329013Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:15:53.984463817Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"701acd62-05c0-48c7-8e95-b65c7cbc47b1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65152","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:02.510997581Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"1.72577933s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.915361069Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.915596086Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.915603836Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.93792568Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.937964082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.956427571Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.956463622Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.976016593Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.976045954Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.995362929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:04.99539589Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.013903641Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.013931042Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.033472744Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.033506205Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.053130289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.053172281Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.072812195Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.072856716Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.073078123Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.073185606Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107218779Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107261761Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107270841Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107277531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107283992Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:05.107410225Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"28bbcd91-503f-4c5a-a465-e0a20feb0f60","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized)]","latency_us":192452,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:05.107612741Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"13.214666717s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28bbcd91-503f-4c5a-a465-e0a20feb0f60\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:16:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"28bbcd91-503f-4c5a-a465-e0a20feb0f60\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.287232493Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.287475884Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.287482594Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.308473319Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.30851571Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.326462794Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.326493345Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.34611909Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.346158711Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.363513337Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.363545538Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.382540614Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.382580455Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.401017144Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.401049325Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.418314059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.41834874Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.436125379Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.43616097Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.436447469Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.436587613Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467218915Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467254796Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467263656Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467270426Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467276866Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:18.467414481Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"eb343b3c-d4fd-4a6f-b71c-f0f7c9f00d96","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized)]","latency_us":180545,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:18.467605026Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"20.342145697s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb343b3c-d4fd-4a6f-b71c-f0f7c9f00d96\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:16:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb343b3c-d4fd-4a6f-b71c-f0f7c9f00d96\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:19.228258381Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"4.989220244s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:23.93434108Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:23.934463124Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4dcd296e-f072-441f-b8dc-b3cfb654d99d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52673","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.755404803Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.755732983Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.755744463Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.778111169Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.77813703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.798270869Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.79830723Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.819841791Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.819863992Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.838074344Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.838092924Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.857367488Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.857391709Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.876279981Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.876311582Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.896172303Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.896211844Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.91453475Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.914563761Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.914831699Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.914934442Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.948080009Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.94812575Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.94813461Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.94814122Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.948147871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:38.948236363Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"90117330-6af4-497e-8cde-43aca6505f90","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized)]","latency_us":193208,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:38.948390478Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"21.706019998s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"90117330-6af4-497e-8cde-43aca6505f90\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:16:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"90117330-6af4-497e-8cde-43aca6505f90\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:39.200594656Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"5.514489279s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:53.884799574Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:16:53.884935768Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"581ae24b-490c-4738-bbed-67fc588f6220","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64650","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:16:59.69388235Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"9.878296835s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.596769699Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.597076178Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.597087279Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.618588769Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.61861792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.636533463Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.636559624Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.657072934Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.657101275Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.676345018Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.676368789Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.6962359Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.696267001Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.714865455Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.714904996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.733463439Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.733493249Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.751745563Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.751816545Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.752107554Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.752228447Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.783777066Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.783848959Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.783859609Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.783866649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.783876229Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:00.783994883Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"07808392-1814-43df-ad8a-12a2bfa42bd1","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized)]","latency_us":187598,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:00.784148308Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"27.152081452s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07808392-1814-43df-ad8a-12a2bfa42bd1\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:17:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07808392-1814-43df-ad8a-12a2bfa42bd1\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:23.835647279Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:23.835775513Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"579b18bf-f449-415f-84e0-757dc98f9a6f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59242","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:24.540039629Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"42.962769768s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.863913131Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.86422949Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.86424243Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.886143022Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.886184694Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.90655415Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.906581131Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.92433895Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.924370793Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.944069017Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.944093888Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.964430903Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.964460454Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.983214583Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:27.983243003Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.001127806Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.001166677Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.021172773Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.021215354Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.0214275Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.021559034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.055790213Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.055824614Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.055832844Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.055841005Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.055847515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:28.055942108Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"36ad85df-536b-4719-9dcf-2d1821ffc472","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\", TimeNotSynchronized), (\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\", TimeNotSynchronized), (\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\", TimeNotSynchronized), (\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\", TimeNotSynchronized), (\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\", TimeNotSynchronized)]","latency_us":192423,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:17:28.056072741Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"30.811109056s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36ad85df-536b-4719-9dcf-2d1821ffc472\", \"content-length\": \"124\", \"date\": \"Wed, 06 Nov 2024 00:17:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36ad85df-536b-4719-9dcf-2d1821ffc472\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:53.787739142Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:53.787810454Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ef043c3e-bd23-4b21-a561-5b3a2c79feea","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36999","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.784693771Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.784918908Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.784925308Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.806799539Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.8068458Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.826439213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.826466574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.845560663Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.845579863Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.864111655Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.864132556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.882584775Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.882602185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.901263201Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.901287682Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.920094482Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.920112702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.9378216Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.93784892Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.938088157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:58.93818399Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.020399328Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.020427059Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.020432779Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.020437999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.020441569Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.112582652Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.116629463Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.121180258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.125987061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.130726882Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.13569301Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.139927626Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.144627746Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.149572553Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:17:59.154440288Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.063392218Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.063573653Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.063627925Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.069447938Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.082884178Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.083036713Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.095344879Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.095457202Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.134248687Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:02.134456033Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 52.74107ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.210840787Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 56.1843ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.218729402Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 64.21234ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.226264926Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 45.204504ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.233573434Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 33.168152ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.2764348Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 68.234434ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.342569759Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 136.718898ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.390448814Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 109.903741ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.455318045Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 74.983255ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:02.664036319Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 82.8355ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:03.249143008Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 125.189593ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:03.296960371Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 251.276595ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:03.361539194Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 246.364229ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:03.588177451Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 149.23577ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:03.778579549Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 294.704815ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:04.003764282Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 316.18197ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:04.161386875Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 509.662732ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:04.479575147Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 292.184414ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.011623696Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 239.242425ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.032928131Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 353.164627ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.185496673Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 926.039388ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.199721718Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 758.50677ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.212455505Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 530.477297ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.223198305Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 555.980304ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.283347686Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 1.179296664s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.549298053Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 867.095107ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.76316234Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 611.944102ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.848268713Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 707.922512ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:05.979258623Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 1.068599588s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:06.133771763Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 1.003897649s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:06.467497918Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.141986347s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:06.63686759Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 956.176699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:06.694020271Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 999.546748ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:06.735723903Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 1.053590762s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:07.209212059Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 822.125132ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:07.4793165Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 1.183237002s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:07.658069001Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 1.492468772s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:07.744440562Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.320385701s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:07.789827473Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 565.972829ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:08.27106402Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 994.086495ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:08.310858884Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 1.044639986s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:08.845744098Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 896.505772ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:08.854402616Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 969.108659ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.114345264Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 711.100192ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.24170925Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 1.411103081s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.309790983Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 1.212281484s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.756986666Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 1.19692108s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.894928531Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 1.229458811s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:09.959071862Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 657.264265ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:10.088252467Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920\") failed: Property not found. retry in 1.473154173s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:10.727193509Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.373355336s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:10.753241474Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6\") failed: Property not found. retry in 1.146718458s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:10.977531261Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb\") failed: Property not found. retry in 741.62532ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:11.098976537Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600\") failed: Property not found. retry in 1.476111663s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:11.195116069Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 766.764635ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:12.1471095Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.449859478s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:12.922188104Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 651.924206ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:14.379661873Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.119229332s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:15.464749608Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 906.79697ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:16.594053175Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 969.408054ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:17.511789486Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 872.597401ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:18.490152653Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 777.875909ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:19.372950332Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.070407473s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:20.160344529Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.238296457s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:21.240029824Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:22.393922513Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"55.786129445s"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 861.745765ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:22.488507141Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf\") failed: Property not found. retry in 1.278388337s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:23.361752917Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:23.813774157Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:23.813811688Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"05a913b0-6d88-44ec-b944-626956398f5c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33015","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"200"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663499635Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_e96b89a0-b522-4f27-b569-f9afd2cf64bb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663568717Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_b4d90277-34a2-44b2-b9aa-ebe154ede1f6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663574687Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_af83424a-6cda-458a-b37a-63cee20a76bf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663579287Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_ee723c3a-1ba1-4fbb-b011-8582e21b8920"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663583577Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_bffbce44-565e-4066-be6c-d2ce82f50600"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.663594717Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.762826904Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.967532581Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8e78095c-9a53-4732-ad4f-fcc4051e30f4","remote_addr":"[fd00:1122:3344:101::1]:60254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26235200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:24.999464249Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60271"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:25.029199812Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3705"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:34.55353128Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3723"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.526150577Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.5262413Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"a882335f-7412-4af9-adb9-684c403c1105","remote_addr":"[fd00:1122:3344:101::1]:60271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20526063,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7102227a-61f8-4dda-a9a0-f6678fae71c6 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 51df9631-cef5-49ae-8cfd-f0c05bdac388 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: ad2fb963-23a1-4187-b186-e731e9dae98d (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f22adc87-bf5c-42cf-9731-7ae698af699f, kind: Service { id: cdbd7d61-5fa7-425e-9630-03453e0bcbbe }, name: Name(\"ntp-cdbd7d61-5fa7-425e-9630-03453e0bcbbe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 94])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ee723c3a-1ba1-4fbb-b011-8582e21b8920 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: bffbce44-565e-4066-be6c-d2ce82f50600 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: af83424a-6cda-458a-b37a-63cee20a76bf (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: e96b89a0-b522-4f27-b569-f9afd2cf64bb (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 309c9aef-da72-4f08-9f70-c20d80316472 (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e2378399-e8ed-453c-847a-0a419dfd0966 (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e (service), filesystem_pool: Some(ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5f3a2ed9-e967-419d-8613-7c57d709adb5 (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa896f4b-4691-418e-8e9e-8b3c70d28865 (service), filesystem_pool: Some(ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 25723913-943d-452f-bab1-aba6e0bbedd3 (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: f0225290-2d83-4eac-a3eb-4816abc42528 (service), filesystem_pool: Some(ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: dacd6543-b9fc-4eee-840b-3a7657eebade (service), filesystem_pool: Some(ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External } } } }, OmicronZoneConfig { id: f0b30f62-cb14-417f-a5b2-f6887992fefe (service), filesystem_pool: Some(ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9e32dc68-c273-4519-b053-5b4fe091230c (service), filesystem_pool: Some(ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External } } } }, OmicronZoneConfig { id: f18d6d96-6447-45c4-9068-2f092d9ddca6 (service), filesystem_pool: Some(ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea326512-acec-46ba-8493-6cb6258d603a (service), filesystem_pool: Some(ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 18eb641c-5db4-446b-8d03-6d8e7d30d926 (service), filesystem_pool: Some(ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External } } } }, OmicronZoneConfig { id: e042603d-2535-467c-922f-95095b6b9ae4 (service), filesystem_pool: Some(ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12bc462-06cf-42fc-818e-35378a8f17a1 (service), filesystem_pool: Some(ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External } } } }, OmicronZoneConfig { id: 943cbdf5-af10-4962-803b-9403bc5e104f (service), filesystem_pool: Some(ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.562326811Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.562485797Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53681"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.562802906Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.562809497Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7102227a-61f8-4dda-a9a0-f6678fae71c6, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.588402505Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.588448977Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 51df9631-cef5-49ae-8cfd-f0c05bdac388, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.611060268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.611099369Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ad2fb963-23a1-4187-b186-e731e9dae98d, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.632961108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.632994479Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ee723c3a-1ba1-4fbb-b011-8582e21b8920, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.654035744Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.654067365Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bffbce44-565e-4066-be6c-d2ce82f50600, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.674239213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.674270054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: af83424a-6cda-458a-b37a-63cee20a76bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.694211486Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.694253048Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e96b89a0-b522-4f27-b569-f9afd2cf64bb, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.713069856Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.713101357Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b4d90277-34a2-44b2-b9aa-ebe154ede1f6, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.731537559Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.731563716Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 309c9aef-da72-4f08-9f70-c20d80316472, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.847529288Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.847568859Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 309c9aef-da72-4f08-9f70-c20d80316472, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f3a2ed9-e967-419d-8613-7c57d709adb5, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.9550422Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:45.955098691Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5f3a2ed9-e967-419d-8613-7c57d709adb5, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dacd6543-b9fc-4eee-840b-3a7657eebade, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.060266634Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.060316215Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: dacd6543-b9fc-4eee-840b-3a7657eebade, dataset_name: DatasetName { pool_name: ZpoolName { id: f049eab6-1590-47c8-b087-bc798a2cfc73 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0b30f62-cb14-417f-a5b2-f6887992fefe, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.153065268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.15310746Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f0b30f62-cb14-417f-a5b2-f6887992fefe, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a4fd05b-1fb2-49d1-ab40-b32093e6fd86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e32dc68-c273-4519-b053-5b4fe091230c, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.270156295Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.270206086Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9e32dc68-c273-4519-b053-5b4fe091230c, dataset_name: DatasetName { pool_name: ZpoolName { id: 25d63394-e252-4651-bd2a-cb7cb7ea5b3e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f18d6d96-6447-45c4-9068-2f092d9ddca6, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.388943261Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.388992623Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f18d6d96-6447-45c4-9068-2f092d9ddca6, dataset_name: DatasetName { pool_name: ZpoolName { id: aaf2d272-1feb-4e24-ba54-7075c6ab94d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ea326512-acec-46ba-8493-6cb6258d603a, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.499367249Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.499430131Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ea326512-acec-46ba-8493-6cb6258d603a, dataset_name: DatasetName { pool_name: ZpoolName { id: b89e85b3-c4ff-4874-bdbd-5e8cca783de4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18eb641c-5db4-446b-8d03-6d8e7d30d926, dataset_name: DatasetName { pool_name: ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.5937152Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.593761171Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 18eb641c-5db4-446b-8d03-6d8e7d30d926, dataset_name: DatasetName { pool_name: ZpoolName { id: a62f7ed5-f062-4f65-8692-31c405e42f70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e042603d-2535-467c-922f-95095b6b9ae4, dataset_name: DatasetName { pool_name: ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.691384419Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.691432051Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e042603d-2535-467c-922f-95095b6b9ae4, dataset_name: DatasetName { pool_name: ZpoolName { id: 392a3437-26be-4bcb-8f19-18fd4492b7e0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f12bc462-06cf-42fc-818e-35378a8f17a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.772868999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.77290945Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f12bc462-06cf-42fc-818e-35378a8f17a1, dataset_name: DatasetName { pool_name: ZpoolName { id: 60f04be8-23a0-4089-b45c-357c38441897 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 943cbdf5-af10-4962-803b-9403bc5e104f, dataset_name: DatasetName { pool_name: ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.863121508Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.863164239Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 943cbdf5-af10-4962-803b-9403bc5e104f, dataset_name: DatasetName { pool_name: ZpoolName { id: 0ff26524-4a22-4458-bf83-380365ee3e3b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.959980543Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.960031295Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.960243731Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:46.960392246Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.069197946Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.069447213Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.116532991Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.116746917Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.116810599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.116861601Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.15154661Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.153853819Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.15392358Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.153942152Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.329822913Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369265414Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369535962Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369552992Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369565473Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369595854Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.369599824Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.370735668Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.382737034Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.382990591Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.38933928Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.39542408Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.4004596Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.405106708Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.409380515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.414659292Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.420106983Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.54530475Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.551124803Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.556658387Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.560928804Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.56618866Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.571691513Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.577102344Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.582647739Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.588551734Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.593913953Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.599645003Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.605169553Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.610144515Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.61772462Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.665843008Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.666403875Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.674611048Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.681459552Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.68781258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.69452262Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.702896988Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.77708484Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.887029242Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.94247803Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.944279714Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:47.944380667Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.000084541Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 244, 168]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.000121302Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.076221161Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.078586861Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.078604032Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.16141835Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 204, 8]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.161450121Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.169667185Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.169687986Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.249378372Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 157, 162]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.249412033Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.250173535Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.250235347Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.252054521Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.252099102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306389864Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 215, 111]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306418255Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306621801Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306633561Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306690513Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.306695133Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.308371443Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.308398384Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.361295804Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 166, 251]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.361322335Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.361597033Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.363323774Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.363444698Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.365068736Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.366959542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.366973493Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.521279973Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.538397172Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.538468714Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.53900677Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.551238343Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.562305562Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.562385894Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.562619197Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.566376512Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Installing Omicron zone: oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.566673181Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.618298093Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.626769415Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.635142403Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.639905225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.648914962Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Installing Omicron zone: oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.955427912Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:48.990757681Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.04395109Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.339897366Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.340096062Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.62902794Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.629260786Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.683352303Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.683545538Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.774744776Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.774917051Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.783446784Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.78362774Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.796920464Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.79711289Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.806237441Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.806524539Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.825893664Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.826116901Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.845405964Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.845635441Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 34.398254ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:49.870396006Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 67.376685ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:49.88266552Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 25.233587ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:49.893533903Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.893831431Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.894211043Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 80.364396ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:49.917548456Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.918091722Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:49.918332909Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 75.366138ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:49.938289261Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 88.458417ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.008943729Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 103.754833ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.03290447Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 48.86388ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.051454851Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 31.195177ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.067788682Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 39.249399ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.090645924Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 212.76356ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:50.173701241Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:50.173964118Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:50.174529705Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:54.186985229Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57555"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:54.187786333Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9099dae2-4518-48c7-ba36-e11b3ac0d941","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57555","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":202,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 199.974593ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.207822528Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 54.280333ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.257756552Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 315.922364ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.291496552Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 140.035892ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.318012539Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:54.328855251Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 60.721741ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.339735934Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 146.982093ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.385767451Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 143.792092ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:54.463484338Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:54.464105466Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 482.541686ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.361957008Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.36203771Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 46.367505ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.377037976Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 66.006526ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.393824644Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.394962448Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 396.127607ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.41895915Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 80.78591ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.432348828Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 1.02188252s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.445646743Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 155.422567ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.457215826Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.457295838Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 98.432136ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.476349474Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 250.369302ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.487708561Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 171.479099ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.499682017Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.49979888Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 73.576545ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.519125954Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 86.749658ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.530825552Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 65.934027ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.54223221Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 154.797189ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.552804104Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.553170935Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:58.553315509Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 187.175369ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.587219166Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 90.485848ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.607115156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 63.757565ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.617117043Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 252.539418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.627076618Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 51.417147ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.636503538Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 288.220771ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.649898506Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 219.344137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.664134599Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 367.795619ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.683894456Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 67.028697ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.695368366Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 128.79824ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.705512967Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 41.676853ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.714997429Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 187.024796ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.724850691Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 74.858655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.73559109Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 398.158398ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.758113279Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 230.871973ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.769390024Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 77.514898ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.778949838Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 38.592321ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.787988896Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 97.167258ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.797321373Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 228.608169ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.806854456Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 115.505456ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.819230874Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 464.095985ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.845839703Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 238.126141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.873265208Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 71.023121ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.885955394Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 208.893477ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.896749645Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 1.148068693s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.908480283Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 224.698082ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.91847305Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 340.341269ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.930747284Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 206.921395ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.94272493Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 432.553014ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.952650415Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 252.350264ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.96562612Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 434.241193ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.978961296Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 252.104611ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:58.993980242Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 836.065742ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.015983625Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 602.9567ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.050982914Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 1.142589045s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.06871593Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 464.289359ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.122350703Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 462.359722ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.1357419Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 840.837012ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.177554492Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 1.114602755s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.209860431Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 329.865509ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.292871075Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 708.841241ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.313794337Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 241.378463ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.328930576Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 851.166162ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.341658634Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 619.026418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.355462903Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 879.960889ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.402096018Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.409142587Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.409289861Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 1.421788735s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.431346086Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 1.399425604s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.480949799Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.496356457Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.496503011Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.557637596Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:18:59.557728439Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 776.128521ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.584116992Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 826.260852ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.60225711Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 996.289996ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.616228795Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 1.148339315s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.639782754Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 1.393746179s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.670748164Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 25.969002ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.683170792Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 119.096772ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.721786719Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 41.958427ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.736117584Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 134.906456ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.8269245Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 119.327239ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.898280129Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 61.115472ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.911206453Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 1.421192896s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.923729884Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 160.136658ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:18:59.980040326Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 564.883286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.002425471Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 145.128319ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.015315564Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 525.26827ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.035294307Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 1.21102497s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.051006453Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 1.391006546s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.064796362Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 1.181540151s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.082407555Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 568.96137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.15499846Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 279.967187ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.176507699Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 556.542451ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.209366895Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 1.0915186s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.229527533Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 739.922128ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.300842241Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 1.163273517s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.343904429Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 969.587788ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.377422974Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 1.387553452s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.446779033Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 511.473098ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.470742905Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 608.99068ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.573096903Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 560.12207ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.585077239Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 1.269236142s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.627589021Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 659.496809ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.73803016Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 1.310747365s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.783216402Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 985.84414ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.804179964Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 711.929068ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.870020728Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 878.804169ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:00.895989979Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 620.593551ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.002739649Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 866.284027ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.055174225Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 960.429797ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.078355604Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 1.339047358s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.163862022Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 535.177331ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.201004865Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 1.013888728s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.27864775Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 1.090904128s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.296106069Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 532.695386ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.337951381Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 980.463365ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.363453728Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 1.070372816s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.382603006Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 823.604562ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.419319027Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 1.239259837s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.475241187Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 705.380076ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.528671963Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 1.172865828s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.600942839Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 1.358660222s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.638266136Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 1.46770553s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.752092346Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 1.178237957s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.789705402Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 1.230751859s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.802992297Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 981.890118ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.847483838Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 1.252255313s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.88426934Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 623.718959ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.918028652Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 967.428003ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:01.93950236Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 792.283139ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.055592506Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 798.777474ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.106856268Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 895.655828ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.251532974Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 1.010728666s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.267035964Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 636.293274ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.312882565Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 1.334802607s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.361018514Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 779.65844ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.402122944Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 932.261016ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.468554406Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 651.791399ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.518786768Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 1.168812331s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.555741765Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 1.312230787s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.72769196Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 1.303155427s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.787083543Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 576.719638ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.844992122Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 1.436662986s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.861146422Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 1.324566916s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.922205185Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 928.829029ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.9361823Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 1.087938601s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.963932054Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 1.140930639s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:02.981349941Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 1.049252357s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.013493155Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 1.280400541s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.050102472Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 1.145771651s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.150869984Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 1.357314921s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.165006754Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 555.504458ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.183902755Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 1.211343591s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.195257292Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 521.421886ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.233285201Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 1.001463593s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.293632252Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 1.352640008s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.414316465Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 674.843284ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.435288738Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 545.723233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.710356754Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 1.401417677s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.7452661Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 665.697793ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.759536744Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 943.322996ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.771420107Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 862.183727ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:03.878723242Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 1.166571954s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.051801921Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 689.850542ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.065135087Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 579.51475ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.076320599Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 1.336494432s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.102860867Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 962.137842ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.12049426Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 848.531262ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.135395983Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 934.848664ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.258990092Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 1.223271452s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.270634908Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 565.761631ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.308502322Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 1.173965024s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.322858638Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e\") failed: Property not found. retry in 1.293315545s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.333852204Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 1.258821991s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.347525821Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3\") failed: Property not found. retry in 1.204216263s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.42095181Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 1.290274176s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.436599805Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 999.324078ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.537057848Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 793.301592ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.670967979Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 1.308087425s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.731087288Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 1.178482415s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.754379109Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 1.199264639s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.773762395Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 667.062777ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.791205823Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 1.333417578s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.891515011Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 654.169245ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:04.999100705Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 1.084540071s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.096219148Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 729.843799ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.159134426Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 880.766429ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.208594614Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1\") failed: Property not found. retry in 1.239685212s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.230812194Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe\") failed: Property not found. retry in 1.481973625s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.451544327Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 798.540619ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.47117701Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 712.35459ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.48362587Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a\") failed: Property not found. retry in 1.181433007s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.50855854Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4\") failed: Property not found. retry in 871.834341ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.521115803Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 879.996551ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.549191757Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 1.431355172s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.618293528Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 809.244868ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.679275648Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865\") failed: Property not found. retry in 630.841523ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.739745943Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 919.477285ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.905754282Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528\") failed: Property not found. retry in 989.060716ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.949372167Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6\") failed: Property not found. retry in 1.167544925s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:05.990385204Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 1.408417535s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.05288062Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 913.561991ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.105462391Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 730.79412ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.195918357Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 1.19716185s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.21119199Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 557.126185ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.238899953Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 809.310064ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.283180477Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade\") failed: Property not found. retry in 1.306302098s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.44561379Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c\") failed: Property not found. retry in 1.291507662s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.506425975Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 868.13577ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.815021707Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926\") failed: Property not found. retry in 1.296905171s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.84106758Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 616.320772ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:06.941933415Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f\") failed: Property not found. retry in 1.077649727s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.039351327Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 534.76463ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.064662869Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966\") failed: Property not found. retry in 1.075558456s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.105917253Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 1.168439638s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.420769481Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 712.24151ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.480065921Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\") failed: Property not found. retry in 1.202024116s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.571778514Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472\") failed: Property not found. retry in 648.632959ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.612774951Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 1.305823123s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:07.697109665Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5\") failed: Property not found. retry in 1.027888266s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:08.231038277Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a\") failed: Property not found. retry in 1.434175692s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:08.61145343Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9\") failed: Property not found. retry in 750.467159ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:09.018795524Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076256818Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_aa896f4b-4691-418e-8e9e-8b3c70d28865"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076359871Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_f13e9b98-ae91-4d61-ba9b-3b178cfa151a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076371582Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_dacd6543-b9fc-4eee-840b-3a7657eebade"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076396292Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_9e32dc68-c273-4519-b053-5b4fe091230c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076409823Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_f0b30f62-cb14-417f-a5b2-f6887992fefe"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076421103Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_309c9aef-da72-4f08-9f70-c20d80316472"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076431633Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_clickhouse_5f3a2ed9-e967-419d-8613-7c57d709adb5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076442454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_e042603d-2535-467c-922f-95095b6b9ae4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076449994Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_ea326512-acec-46ba-8493-6cb6258d603a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076460094Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_oximeter_4e1b9b8b-9e08-4e6d-89c1-97aff5e8f01e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076469245Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_9bf7f7e3-0a6c-48b6-84be-c4bb8425f761"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076475675Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_f0225290-2d83-4eac-a3eb-4816abc42528"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076482015Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_25723913-943d-452f-bab1-aba6e0bbedd3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076488995Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_18eb641c-5db4-446b-8d03-6d8e7d30d926"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076505196Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_f18d6d96-6447-45c4-9068-2f092d9ddca6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076512016Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_f12bc462-06cf-42fc-818e-35378a8f17a1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076521866Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_e2378399-e8ed-453c-847a-0a419dfd0966"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076528896Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_f633068a-05f8-4be2-a65b-d83ac560f7a9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.076535867Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_943cbdf5-af10-4962-803b-9403bc5e104f"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.080998639Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.08506414Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.090875762Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1f1b7704-1b8e-4312-8368-d4a7bf96f89f","remote_addr":"[fd00:1122:3344:101::1]:53681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":24527673,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.100833518Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.100851409Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.101054675Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.103078745Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:708"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.103203718Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.144521415Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.145486324Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.247879714Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.389314903Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.559803174Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.654172345Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.794554753Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:10.980919076Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:11.271289397Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:11.500625826Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:11.88003743Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:12.316473687Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:12.865123865Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:13.764763485Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:14.240273482Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:14.946064556Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:16.78900097Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:19:18.214117961Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"71.69866798s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/3a58cbfb-5015-41c2-9142-616f1a1558aa/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:18.604267724Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d34d7dc7-0ffc-40e8-a07c-ea01efc68fa5\", \"content-length\": \"173\", \"date\": \"Wed, 06 Nov 2024 00:19:23 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"3a58cbfb-5015-41c2-9142-616f1a1558aa\\\"\", request_id: \"d34d7dc7-0ffc-40e8-a07c-ea01efc68fa5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:23.355117231Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:24.236241911Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:24.236486238Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ebd03a1c-b4f0-47d8-b710-c7eaf9f6a03c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64981","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06ee5dbc-8d40-4973-8b95-73fb82e78b30\", \"content-length\": \"173\", \"date\": \"Wed, 06 Nov 2024 00:19:24 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"3a58cbfb-5015-41c2-9142-616f1a1558aa\\\"\", request_id: \"06ee5dbc-8d40-4973-8b95-73fb82e78b30\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:24.523333604Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ecca7eb-86c2-4478-9427-838f19f38122\", \"content-length\": \"173\", \"date\": \"Wed, 06 Nov 2024 00:19:27 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"7c647f25-eb29-494d-840d-11658133af45\\\"\", request_id: \"7ecca7eb-86c2-4478-9427-838f19f38122\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.314896751Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.412569191Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.509101127Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50666"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.509768707Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.509921941Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.510115137Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.510417776Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:28.713943838Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"6f45a96f-d2e6-447f-9557-7a431d0e879d","remote_addr":"[fd00:1122:3344:101::c]:50666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":204170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.681790527Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53854"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682425116Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682506479Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.6825441Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.6825651Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682587991Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682617432Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682634052Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.682687674Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6daa7edb-44ab-43dc-bfef-22f39c26a7f3","remote_addr":"[fd00:1122:3344:101::b]:53854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":722,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:33.683468257Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.051251526Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59439"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.051405691Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.051411731Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.051448362Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.05172397Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.242956328Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"03f9db99-1449-4804-a738-3b3f759e9ecc","remote_addr":"[fd00:1122:3344:101::b]:59439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191581,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.493948012Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35143"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494252599Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.49432249Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494360392Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494379612Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494400263Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494427854Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494440364Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.494459445Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"04a9e68e-980c-4189-8239-483fda0e8bac","remote_addr":"[fd00:1122:3344:101::b]:35143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":383,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686430764Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43620"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686618249Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686671881Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686688341Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686713472Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686718972Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686726562Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686740613Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.686764854Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"af955a3c-24ea-4931-b0c1-5dbd95f5a199","remote_addr":"[fd00:1122:3344:101::a]:43620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":269,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740381885Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:65315"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740580111Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740628393Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740647103Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740682814Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740700905Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740721935Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740730756Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:34.740754236Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"35e8c226-24ff-4620-8a06-6948fe09fb93","remote_addr":"[fd00:1122:3344:101::c]:65315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":276,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:41.525845519Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:41.525993193Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a9dd64d8-e168-4a0d-bd49-02d9333cb93b","remote_addr":"[fd00:1122:3344:101::b]:54158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:41.606623867Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:41.606649828Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c1a8da01-e72c-4c80-8b6c-262e45a7bcad","remote_addr":"[fd00:1122:3344:101::b]:54158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:41.709028057Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4635996c-9876-4a6f-b8b3-1fd2bddd14da","remote_addr":"[fd00:1122:3344:101::b]:54158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2071,"response_code":"204"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:47.448312483Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:54.270560943Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:54.270730428Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"54d75d2f-25a3-4b5a-a6e1-4fafac7f1dde","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38476","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.401382597Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.401526321Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b02cdd81-b6fb-429b-8804-213304dfc445","remote_addr":"[fd00:1122:3344:101::a]:63073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.476747414Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.476784965Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ee846320-daf4-4cfd-aeab-4f7d5e78c4e0","remote_addr":"[fd00:1122:3344:101::a]:63073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.508406754Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.508667062Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d642accf-7b2e-42fe-b2e0-5438b82a4842","remote_addr":"[fd00:1122:3344:101::a]:52049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.516177055Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"10052a60-425a-491d-b7b8-5056cc8ce07a","remote_addr":"[fd00:1122:3344:101::a]:63073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.551023929Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38346"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.551189634Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.628715536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.6888136Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ba963a15-5f5b-4708-90e8-b441092c3d95","remote_addr":"[fd00:1122:3344:101::a]:38346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":137714,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.691669935Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"52878157-cb9d-4555-908f-43e001d23719","remote_addr":"[fd00:1122:3344:101::a]:38346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.724368856Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.724475769Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c8f0f035-81cf-42ce-985b-fd1481b9e0c4","remote_addr":"[fd00:1122:3344:101::c]:39299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.739496395Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.739623519Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.740758222Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.741796253Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.741814354Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"d62b705f-718c-47f4-b137-cb9d4e96bf7d","remote_addr":"[fd00:1122:3344:101::a]:38346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2615,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.770332631Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.770348911Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f1e590dc-ecba-4d0d-bb05-ba229fd57e7c","remote_addr":"[fd00:1122:3344:101::c]:39299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.801829596Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.80198496Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"74149a56-67cb-486f-bd9f-f151eb418493","remote_addr":"[fd00:1122:3344:101::c]:39769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.808820963Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cb1b1523-c6f4-4fef-a04f-270ce45f81ca","remote_addr":"[fd00:1122:3344:101::c]:39299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":261,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.91649142Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.916611474Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"364ff9da-3415-4986-82b2-18be17c8628c","remote_addr":"[fd00:1122:3344:101::b]:59585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.983642304Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.983678125Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1832f357-4b56-4069-8167-58215d1111d3","remote_addr":"[fd00:1122:3344:101::b]:59585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":73,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.983704326Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37421"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:55.983789738Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.030610258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.096049931Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6017a48e-98be-45c3-83ad-b457c0d5bd63","remote_addr":"[fd00:1122:3344:101::c]:37421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112281,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.096461883Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"528b4238-df89-4f98-aa4a-c87c54dfa4b2","remote_addr":"[fd00:1122:3344:101::c]:37421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.096534555Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"dd6d2597-bc0b-4d12-8350-b496bf1e67da","remote_addr":"[fd00:1122:3344:101::b]:59585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":272,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.146629903Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.146856149Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"78bd2e16-202c-4c2c-9390-63d6b7737f10","remote_addr":"[fd00:1122:3344:101::b]:45968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.214292792Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40592"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.214403355Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.25398105Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.309816498Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"61e1ed29-3ec7-4cb7-a449-9c0fb9640d8d","remote_addr":"[fd00:1122:3344:101::b]:40592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95461,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:19:56.312180598Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e67c8ce5-351d-413f-8b24-a84f6554878c","remote_addr":"[fd00:1122:3344:101::b]:40592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:24.30340601Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34304"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:24.303673058Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2071316f-9457-4222-9c44-9d818afbe3ff","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34304","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.4020978Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40213"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.402225284Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5f253a33-370e-4bbd-920c-dffd29428ec1","remote_addr":"[fd00:1122:3344:101::a]:40213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.417159237Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.417182408Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1c1d3bbd-4ead-4612-a859-9de8ff04e38d","remote_addr":"[fd00:1122:3344:101::a]:40213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.507114328Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.507323704Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"145a072b-4e92-4878-85bf-08fb197e39c7","remote_addr":"[fd00:1122:3344:101::a]:58977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.513790706Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2a50d98d-b4a6-4101-be5f-65cdda8c804a","remote_addr":"[fd00:1122:3344:101::a]:40213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.706634321Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59953"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.706755025Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.748736471Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.808457644Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"39a0a606-bdb5-48f1-92ba-0c9e54828e20","remote_addr":"[fd00:1122:3344:101::a]:59953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101744,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.808721422Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.809021641Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8d604b4a-424f-405d-8f3d-d8b7c2ea1e2c","remote_addr":"[fd00:1122:3344:101::c]:41330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.809082263Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"51ba9842-d653-4496-9a3d-14d8b2b072a1","remote_addr":"[fd00:1122:3344:101::a]:59953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":320,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.853566824Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:25.853601075Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b6311cb7-6158-42d0-8b0b-97ad388aeaa0","remote_addr":"[fd00:1122:3344:101::c]:41330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.040170314Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.040345319Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1fff0dd0-d5e9-4c0b-b625-07696304c0db","remote_addr":"[fd00:1122:3344:101::c]:64107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.044214384Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e17de233-4ff0-4232-991b-c5131c060052","remote_addr":"[fd00:1122:3344:101::c]:41330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.16265397Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.162779054Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"082eda8d-8d83-49fa-9d76-f34a952d53ed","remote_addr":"[fd00:1122:3344:101::b]:60060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.23976218Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46796"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.239890294Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.279781408Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.338961885Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dd1230cb-c464-4490-9fff-a49c81d63ab9","remote_addr":"[fd00:1122:3344:101::c]:46796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99124,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.339114469Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.33912789Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8b528679-adaa-4de2-84f9-232007988c65","remote_addr":"[fd00:1122:3344:101::b]:60060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.339262174Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e65d8a37-3717-4f86-bbae-d212c600a255","remote_addr":"[fd00:1122:3344:101::c]:46796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.367335677Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.367596225Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"7a7f4e18-dee2-4bdf-82d2-8c9e822964d0","remote_addr":"[fd00:1122:3344:101::b]:60188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":170,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.376834239Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"486b8d7d-9a44-438c-a5a1-11bd42a48519","remote_addr":"[fd00:1122:3344:101::b]:60060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":284,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.397595776Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38899"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.397720289Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.431021418Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.486687481Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e14cf79e-01dc-403a-ae78-3b6fc9c7aba2","remote_addr":"[fd00:1122:3344:101::b]:38899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89025,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:26.490174864Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0c9b7675-dac7-4aad-a520-ce18632f56cd","remote_addr":"[fd00:1122:3344:101::b]:38899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:36.566764942Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:58224","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:40761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:36.614412476Z","hostname":"sock","pid":1572,"uri":"/7c647f25-eb29-494d-840d-11658133af45","method":"GET","req_id":"612a8935-6201-4f46-b9a6-0be7e5ff9274","remote_addr":"[fd00:1122:3344:101::d]:40761","local_addr":"[fd00:1122:3344:101::1]:58224","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47529,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:54.33567408Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:54.33598921Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"df8ebb11-fd83-453a-85fe-561f552af556","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40633","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.311524143Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.311670837Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e645fcd1-9286-4379-8bb5-dfd0242efd83","remote_addr":"[fd00:1122:3344:101::a]:63754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.327874148Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.327898169Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d7c3aec2-9a32-42aa-87c8-b87366286fcc","remote_addr":"[fd00:1122:3344:101::a]:63754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.386823078Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.387078276Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"930c915e-82c7-4026-84bc-891d8460cf18","remote_addr":"[fd00:1122:3344:101::a]:56343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.396387302Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"82767442-4f0d-4203-bd25-4bd5620edec3","remote_addr":"[fd00:1122:3344:101::a]:63754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.437510343Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57791"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.437615126Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.479209081Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.558794354Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1858f19e-7bdc-4b3d-b196-a5ef000f2f1d","remote_addr":"[fd00:1122:3344:101::a]:57791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":121208,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.559196876Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f0286728-7aa3-4aaa-9d2e-f783fa30ff5d","remote_addr":"[fd00:1122:3344:101::a]:57791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.687477924Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.68765162Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3e72dbc9-b264-46c6-862c-63a5d4c67633","remote_addr":"[fd00:1122:3344:101::c]:61111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.733285244Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.733313655Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e3607a62-0a1a-497f-bb08-5d7d1a736c64","remote_addr":"[fd00:1122:3344:101::c]:61111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.793036158Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.793232654Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"48045806-62df-43d4-a501-7aa9307748b3","remote_addr":"[fd00:1122:3344:101::c]:34305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.801719286Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ed81a345-2363-4832-9747-8fc57cd7ef13","remote_addr":"[fd00:1122:3344:101::c]:61111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":262,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.83080121Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33804"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.830907133Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.88200782Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.956896973Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d5b41f02-9a59-4c52-9a3c-93e7f98962fa","remote_addr":"[fd00:1122:3344:101::c]:33804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126009,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:55.957297455Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0e2bf0d4-5659-482c-8117-39703ec20601","remote_addr":"[fd00:1122:3344:101::c]:33804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.0177544Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.017910655Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a6c6378e-3235-45a4-9a58-8a85482f8d1b","remote_addr":"[fd00:1122:3344:101::b]:40407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.074154185Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.074208066Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c83522f7-a429-4545-afd1-31761a4898d6","remote_addr":"[fd00:1122:3344:101::b]:40407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.083236954Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"502012cd-503c-4cdb-9cdf-2b0cc3272236","remote_addr":"[fd00:1122:3344:101::b]:40407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.143595096Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42546"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.143768802Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"910401c0-4501-454c-b73e-e74002b70d66","remote_addr":"[fd00:1122:3344:101::b]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.176213395Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56480"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.176310868Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.216208232Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.273470522Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b1d713ff-cbc3-438e-b7fd-bc6f965b88f2","remote_addr":"[fd00:1122:3344:101::b]:56480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97209,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:20:56.273775691Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c2c6fc93-b72e-497d-9c77-e00319ee4b0d","remote_addr":"[fd00:1122:3344:101::b]:56480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:24.370456027Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:24.370620282Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"8d508209-6649-4e9b-bd2c-bf791098cdcc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54629","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.485191463Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.48544805Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"059f2f34-4be5-460e-a083-99a1e7e0a522","remote_addr":"[fd00:1122:3344:101::a]:47066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.584150281Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.584279695Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9716fb3d-7abf-4e33-9490-ea4beaf31145","remote_addr":"[fd00:1122:3344:101::a]:59432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.65048396Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.650526531Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"217719fd-9539-499d-b54c-fd23bdc099a4","remote_addr":"[fd00:1122:3344:101::a]:59432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.662003812Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d38b30b2-3370-4880-a657-1537a8c2ed82","remote_addr":"[fd00:1122:3344:101::a]:59432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":227,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.796616919Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35090"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.796775663Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.851422615Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.955260889Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"25197822-7f78-48f0-b5e1-9df71e0ba7b3","remote_addr":"[fd00:1122:3344:101::a]:35090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158495,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.955507906Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33077"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.955798534Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d2d3478f-5340-4f65-b407-384d3c78a154","remote_addr":"[fd00:1122:3344:101::a]:35090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":271,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.955822785Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e67bb24b-00ba-48ab-a220-524e7c19ae97","remote_addr":"[fd00:1122:3344:101::c]:33077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.974283763Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.974319654Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d8e83e4c-5619-48d1-b534-bfde26dd2382","remote_addr":"[fd00:1122:3344:101::c]:33077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.987589228Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56896"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.987797604Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"79b07ef9-93e1-48a2-8034-a62370e33c14","remote_addr":"[fd00:1122:3344:101::c]:56896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":156,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:25.994600977Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7aa6af84-bc45-416b-91d1-4cb43ab41c75","remote_addr":"[fd00:1122:3344:101::c]:33077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.02772692Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44613"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.027833913Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.067840821Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.128719828Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d9c71a95-a036-4c1c-be9a-925cb4038c69","remote_addr":"[fd00:1122:3344:101::c]:44613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100925,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.129051128Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"95386736-8977-472a-92e7-9300feda392a","remote_addr":"[fd00:1122:3344:101::c]:44613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.174667453Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.174830797Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5dcb673f-7437-4549-a93a-c78474c46cc4","remote_addr":"[fd00:1122:3344:101::b]:36029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.188556915Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.188572876Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"59c6f445-cecb-4ce5-969a-5fcc3fc5b03d","remote_addr":"[fd00:1122:3344:101::b]:36029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.199926213Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"de141726-51c5-4092-bbc6-43365f6c3b43","remote_addr":"[fd00:1122:3344:101::b]:36029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":252,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.298164129Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.298317474Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"867f8198-b154-4ba1-ae02-643c45a2bcc5","remote_addr":"[fd00:1122:3344:101::b]:57858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.500746604Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48473"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.500838967Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.539668929Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.600024661Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a7b20271-5084-48dd-b97d-f884f47de2c0","remote_addr":"[fd00:1122:3344:101::b]:48473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99218,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:26.600377822Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f2e2dfeb-ed09-4d9a-8c33-3b771b73d5dc","remote_addr":"[fd00:1122:3344:101::b]:48473","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:54.403789961Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:54.403894744Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"869a0107-f05a-434a-b60a-5a6eea86364d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62032","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.368150722Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42434"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.368256465Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1a3ce223-feee-4ff8-94d9-8702a3d3c7f3","remote_addr":"[fd00:1122:3344:101::a]:42434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.497730439Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.497973417Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"f7ef3539-611c-4f08-b159-43458679dee5","remote_addr":"[fd00:1122:3344:101::a]:51711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.512303922Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59589"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.512415635Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.563185623Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.62914596Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3121470b-02bf-48cc-9d3a-22322ea94eb3","remote_addr":"[fd00:1122:3344:101::a]:59589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116769,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.629376438Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.629398488Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5d7ed556-62a2-4443-a53c-6a30a4aa859d","remote_addr":"[fd00:1122:3344:101::a]:42434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.629600254Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ed6b4d9c-e563-484a-8866-3afe1f65472a","remote_addr":"[fd00:1122:3344:101::a]:59589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.639358284Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8318c634-6bbe-4de5-a9ac-5ac3fb732499","remote_addr":"[fd00:1122:3344:101::a]:42434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.720630207Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.72073731Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3327ba3e-2278-4739-b4aa-e221de449c56","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.744497385Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.744683831Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1375d2cf-05b9-4fb0-8da3-f05514c09f52","remote_addr":"[fd00:1122:3344:101::c]:39241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.891725947Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.891753707Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e639c150-1063-4714-aa10-612b0627f546","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.900985411Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1064cdf7-c490-4a0e-bda1-e5580ecd9f48","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.941130093Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.941245247Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"acc591d4-5392-4f0f-9cfb-47b1d9b69631","remote_addr":"[fd00:1122:3344:101::b]:47321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.997454166Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:55.997480136Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ae867848-c42d-4daf-aa56-ba91236e3da2","remote_addr":"[fd00:1122:3344:101::b]:47321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.009255686Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e38e698b-d223-40ea-b8fa-fc27c8f9513b","remote_addr":"[fd00:1122:3344:101::b]:47321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.095189127Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:65462"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.095333881Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0c7bdf53-f8ed-4990-882e-cd2c3926f508","remote_addr":"[fd00:1122:3344:101::b]:65462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.123152847Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38461"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.123218449Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.164079733Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.224447625Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e8416e8b-07c1-41ad-b2dd-3d6511c5f997","remote_addr":"[fd00:1122:3344:101::b]:38461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.224592139Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60911"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.224794325Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.228882157Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"79078c3c-ad7c-4d05-9987-500bed885af5","remote_addr":"[fd00:1122:3344:101::b]:38461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4109,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.261306069Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.320727364Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5121b9ef-21b3-4c16-aa4d-ae74014aade4","remote_addr":"[fd00:1122:3344:101::c]:60911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96029,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:21:56.321167226Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"21b86994-f18e-4ccc-ad4f-eaad831df91d","remote_addr":"[fd00:1122:3344:101::c]:60911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:24.436614566Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:24.436786341Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"35b334c6-dbb8-4dbb-96e0-3f6d31e24271","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46151","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.4185595Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61996"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.418710894Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2410efe6-557a-4018-8fbb-20f72b4c48e9","remote_addr":"[fd00:1122:3344:101::a]:61996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.464531874Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.464560515Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1e56aa9f-eec0-48d0-8680-de09cc0a76c4","remote_addr":"[fd00:1122:3344:101::a]:61996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.474011736Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"54b791fc-73e3-4e0c-a3f1-dc5edb7b6143","remote_addr":"[fd00:1122:3344:101::a]:61996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.603403348Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.603645895Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a29df951-e0c7-468f-8b9f-2373ce8aa7d5","remote_addr":"[fd00:1122:3344:101::a]:36741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":184,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.794461239Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.794577133Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"879d8f06-ff97-4f75-8326-bb703c3f7554","remote_addr":"[fd00:1122:3344:101::c]:53483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.824316606Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38312"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.82444976Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.884132852Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.967750804Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5945c6fd-0e33-4490-9b6e-7393f10705d5","remote_addr":"[fd00:1122:3344:101::a]:38312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143325,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.9679569Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.968221638Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"30a1a354-d50b-491d-9e59-3d04bf842316","remote_addr":"[fd00:1122:3344:101::a]:38312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":176,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.968345952Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.968399213Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"75525f95-b816-43af-803b-866b003ed70c","remote_addr":"[fd00:1122:3344:101::c]:49525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":263,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.968414504Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b353c53b-7785-4fab-af81-de3029551132","remote_addr":"[fd00:1122:3344:101::c]:53483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":242,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:25.980198394Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f0e83fc0-3c8c-4147-9612-877ae8806fdd","remote_addr":"[fd00:1122:3344:101::c]:53483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.128204968Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45083"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.128299641Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1134e0e1-d937-4db2-a90f-039c431248e9","remote_addr":"[fd00:1122:3344:101::b]:45083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.14408292Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.14409883Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1a4fcb87-7f49-4370-9083-f69f7cb0d17c","remote_addr":"[fd00:1122:3344:101::b]:45083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.15185434Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6ed403cb-8ab1-41b4-95d4-bd1d569b0e53","remote_addr":"[fd00:1122:3344:101::b]:45083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.217616563Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.217793218Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a346224a-86eb-41ad-863d-b08fbbdb165e","remote_addr":"[fd00:1122:3344:101::b]:40683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.23502789Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33946"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.235143543Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.284730245Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.36235004Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a59b2d53-3408-4660-b68e-f2c482f07356","remote_addr":"[fd00:1122:3344:101::c]:33946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127241,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.362520575Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42614"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.362741831Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.367271246Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1fb7581c-5faa-4c5a-99fa-e585370301f1","remote_addr":"[fd00:1122:3344:101::c]:33946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4491,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.407699356Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.47721019Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8e9202bc-61fd-48b7-bb16-da3131f3d364","remote_addr":"[fd00:1122:3344:101::b]:42614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114567,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:26.477646473Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1fb8e454-e1c7-4a73-9246-432f907e6889","remote_addr":"[fd00:1122:3344:101::b]:42614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:54.468132552Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:54.468235815Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9b70b03a-f540-484d-b2d9-7c14a30135a2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48256","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.369526833Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.369655627Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3190cb87-5945-42bc-93a6-dbf5ac197e70","remote_addr":"[fd00:1122:3344:101::a]:59200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.397047461Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50813"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.397248596Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e1772752-076c-4ed7-a15e-b38f94f96be3","remote_addr":"[fd00:1122:3344:101::a]:50813","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":122,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.443559582Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.443587772Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"889d138a-8083-47a5-a2ac-7c879ea70f3a","remote_addr":"[fd00:1122:3344:101::a]:59200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.502072119Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60393"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.502193042Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.551534817Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.615010552Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8198f9f9-17a0-4973-8934-a00949dc0a57","remote_addr":"[fd00:1122:3344:101::a]:60393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112862,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.615327941Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d84661d1-506d-4056-9d7c-b9c7d391ec99","remote_addr":"[fd00:1122:3344:101::a]:59200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":212,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.615457735Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"10afcf72-58dd-4d8f-8738-0a0ba8adab80","remote_addr":"[fd00:1122:3344:101::a]:60393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.632926533Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61910"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.633052747Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e5adde6c-04f3-4b1f-9809-d6b93b3323e2","remote_addr":"[fd00:1122:3344:101::c]:61910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.674330213Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.674352873Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"db4f3937-96ad-418e-bec6-c11a3faec2c9","remote_addr":"[fd00:1122:3344:101::c]:61910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.785809723Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b7b902e9-458d-4ab7-b550-a5f7de1396b5","remote_addr":"[fd00:1122:3344:101::c]:61910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.881896915Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.882095071Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"20134e4f-da46-40b8-8666-3f466b0f6715","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.926641494Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.926733416Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e4726591-ffe6-413d-99be-d5a7cbda309b","remote_addr":"[fd00:1122:3344:101::b]:57998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.974462093Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.974497505Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"971c35f3-a8db-48e6-ac30-295588d8b75d","remote_addr":"[fd00:1122:3344:101::b]:57998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:55.984401508Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2aa7e758-7b51-4c81-9bda-987eac62436b","remote_addr":"[fd00:1122:3344:101::b]:57998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.054418967Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38905"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.05451358Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.093701244Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.153477838Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"23270fdc-ae61-4c8d-a1c6-7eea6bca3643","remote_addr":"[fd00:1122:3344:101::c]:38905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98996,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.153588312Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54859"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.153778157Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0ea36e8f-e577-4c63-94e6-79e2cc53a6ba","remote_addr":"[fd00:1122:3344:101::b]:54859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":124,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.153971033Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"20f14c92-3b71-4915-be08-7e62dc24735d","remote_addr":"[fd00:1122:3344:101::c]:38905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.414770286Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52171"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.414924801Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.454156165Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.510642223Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"23a72cce-81c2-4322-b0a8-b4c2804ccb7d","remote_addr":"[fd00:1122:3344:101::b]:52171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95785,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:22:56.510926731Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9b6dfe56-2db3-4c58-a5b2-b27d4b0f5484","remote_addr":"[fd00:1122:3344:101::b]:52171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:24.499757368Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:24.499858971Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"420cf9cc-20a8-40ef-958d-29c5681bf917","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62796","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.546202577Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40359"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.546417013Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e85592f0-3121-4da9-a438-d3b4a5e5f45b","remote_addr":"[fd00:1122:3344:101::a]:40359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.627280604Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.627319535Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"be06a8b4-c325-45da-9b5a-39228eccea4d","remote_addr":"[fd00:1122:3344:101::a]:40359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.719721278Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60043"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.719917144Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4b004c44-3b74-4156-b02a-220c14b813ff","remote_addr":"[fd00:1122:3344:101::a]:60043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.722687006Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45143"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.722774259Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.764338033Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.825944282Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d32ec60e-13e6-4b26-8c9e-e64fffaf31f4","remote_addr":"[fd00:1122:3344:101::a]:45143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.826463507Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2d71e1ec-8e46-4fff-aa4b-6ea1fe619b1b","remote_addr":"[fd00:1122:3344:101::a]:40359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":331,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.826613312Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a817b6a6-781f-4740-bba4-ccfc70632a06","remote_addr":"[fd00:1122:3344:101::a]:45143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":310,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.852575413Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.852769798Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"db878b67-d4d4-4c56-baa0-e6f61f9f0136","remote_addr":"[fd00:1122:3344:101::c]:44742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.872298468Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.872329009Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4d378954-4f9a-4bae-9665-6d032094db60","remote_addr":"[fd00:1122:3344:101::c]:44742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.913409439Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.913606625Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"da21e1d1-da3b-4869-9c1a-8eefd5585e72","remote_addr":"[fd00:1122:3344:101::c]:56266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:25.987288702Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"528b17ae-f13e-4007-9317-7a4913789ec4","remote_addr":"[fd00:1122:3344:101::c]:44742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.151877089Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62482"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.151961111Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.19401749Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.254010371Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"153e5b78-c885-441d-966f-3bd3f2ab8912","remote_addr":"[fd00:1122:3344:101::c]:62482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102072,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.254372292Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e16e1b90-bb2c-4928-8f8c-6c939a07fff8","remote_addr":"[fd00:1122:3344:101::c]:62482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.276983433Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.277173909Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9812a0cc-4148-48f5-beaa-6eb7f29d84f3","remote_addr":"[fd00:1122:3344:101::b]:64591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.319008001Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.319041932Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3fcb247d-d785-4716-9432-c580dea2a98a","remote_addr":"[fd00:1122:3344:101::b]:64591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.330915824Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c0007db1-7a01-4ec8-95d6-e18d88aaf084","remote_addr":"[fd00:1122:3344:101::b]:64591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":357,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.440308862Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50663"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.440520438Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"5d114eb2-0365-48b3-928d-a9eb1b0966d8","remote_addr":"[fd00:1122:3344:101::b]:50663","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.455510403Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43416"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.455610657Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.493669547Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.549747672Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b793c21d-28cd-421e-9f39-937921b0e21d","remote_addr":"[fd00:1122:3344:101::b]:43416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94181,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:26.550084271Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"57414788-224d-4801-b9e3-d2ef3f654c7a","remote_addr":"[fd00:1122:3344:101::b]:43416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:54.532343242Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:54.532506456Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"142398d2-9b23-48a5-8d75-cb79eb7d4737","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35883","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.252465682Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.252634267Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ae11c999-8e9f-4e3c-bd83-369a03bc1296","remote_addr":"[fd00:1122:3344:101::a]:43496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.295164079Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.29519456Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"fc4f3eab-2f9a-432a-b85c-6022794616dd","remote_addr":"[fd00:1122:3344:101::a]:43496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.303363583Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4b4bc1bb-a416-4c20-a552-2b77ee0cbd5e","remote_addr":"[fd00:1122:3344:101::a]:43496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.401647991Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39315"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.401758744Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.439408832Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.497053284Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5ce4be5c-19f6-44be-9915-fc071cc83a2a","remote_addr":"[fd00:1122:3344:101::a]:39315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95344,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.497176817Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.497462096Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"93597c12-b949-4ddb-95be-340ad95934e8","remote_addr":"[fd00:1122:3344:101::a]:58912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.497567729Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"48b44fce-1647-4ef9-8401-883e3a8a15ce","remote_addr":"[fd00:1122:3344:101::a]:39315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":226,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.781195129Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.781314653Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d31206dd-e68f-4f14-8cb9-04ba6bed26cc","remote_addr":"[fd00:1122:3344:101::c]:37127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.828070321Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.828102652Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b3402b3a-320f-4269-a165-5dff86eb2383","remote_addr":"[fd00:1122:3344:101::c]:37127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.849405734Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.849631131Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"bba01fc3-631c-4657-a9d9-1e1f0686c1b6","remote_addr":"[fd00:1122:3344:101::c]:44570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.855884147Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"441a4be9-e0cc-4412-b8a5-a6789ab1a1c4","remote_addr":"[fd00:1122:3344:101::c]:37127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":242,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.926235176Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44890"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.926349709Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:55.974455237Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.040744935Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"620563e2-5e6f-47eb-9a54-43873f268ac8","remote_addr":"[fd00:1122:3344:101::c]:44890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114446,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.043835397Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"45d44473-7661-4ab7-9ff3-6f86f6b1e75b","remote_addr":"[fd00:1122:3344:101::c]:44890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.044989251Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.045186017Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"534bd7da-5f6b-4b90-85db-9aa055388d03","remote_addr":"[fd00:1122:3344:101::b]:35726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.086491514Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.086538625Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3b8ac6fd-246c-4ed7-b329-d2a5946e05e0","remote_addr":"[fd00:1122:3344:101::b]:35726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.102182659Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.102356395Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3afd906f-9930-498f-bcd1-4eb02ee5dd53","remote_addr":"[fd00:1122:3344:101::b]:44614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.105134297Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41729"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.10522944Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.143357062Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.207035523Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"60f616f2-b6e0-4c78-b701-ad1b90c9bf3d","remote_addr":"[fd00:1122:3344:101::b]:41729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101836,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.207419314Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"84a5a1ee-a25c-4d54-a479-1c3fc05ffe64","remote_addr":"[fd00:1122:3344:101::b]:35726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":246,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:23:56.207584179Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"99869d35-2da4-4f7a-8394-170060c1cfc6","remote_addr":"[fd00:1122:3344:101::b]:41729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:24.56621931Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:24.566328643Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e106aecc-6be6-49ea-8e5e-ce1dd3fdeda0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61604","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.455646377Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.455784891Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cabd1ca6-6143-49aa-8847-48ad41421189","remote_addr":"[fd00:1122:3344:101::a]:43647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.503120746Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.503156457Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a15a54f0-1e55-427b-a73d-54e99c146f1f","remote_addr":"[fd00:1122:3344:101::a]:43647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.514407481Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"21c87886-2eff-40a4-9c95-7a748cd92aa5","remote_addr":"[fd00:1122:3344:101::a]:43647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.638883577Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39665"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.639095503Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"645de65c-87fb-40e3-a3ed-5ab349a7ec8a","remote_addr":"[fd00:1122:3344:101::a]:39665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.641133844Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38661"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.641222086Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.683939724Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.74812664Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"985d17bb-5b7f-4614-8cd0-beea202d82f6","remote_addr":"[fd00:1122:3344:101::a]:38661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106937,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748365807Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748728578Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2299ea36-209a-45be-aaac-824c7b7ef12b","remote_addr":"[fd00:1122:3344:101::a]:38661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":289,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.74879907Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748852562Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748882092Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748895073Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748911073Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748925554Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748936424Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.748965835Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"78bb7804-1d7b-4fb7-adec-b4cb62c55559","remote_addr":"[fd00:1122:3344:101::a]:63781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":417,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.795299981Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.795397024Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"38b90288-5089-49d8-ab2b-1266173e9b7d","remote_addr":"[fd00:1122:3344:101::c]:63901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":44,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.812043278Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.8121076Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4b3b8a2f-bec1-4d93-adbe-02df209c0aad","remote_addr":"[fd00:1122:3344:101::c]:63901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.983536949Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55064"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.983734585Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0abe0f59-1e41-41af-b962-3c35131af679","remote_addr":"[fd00:1122:3344:101::c]:55064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:25.994934948Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d0848c27-6203-4670-8ca5-25284bd4e1cd","remote_addr":"[fd00:1122:3344:101::c]:63901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":199,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.102854972Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38617"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.102953235Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.141133728Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.202559452Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dead98c7-1a65-4ae1-a472-9e002dc3580b","remote_addr":"[fd00:1122:3344:101::c]:38617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99627,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.202694736Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58020"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203084077Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203118238Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e4ca0686-2918-41a6-b271-75f98db41bd8","remote_addr":"[fd00:1122:3344:101::c]:38617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":244,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.20314655Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203198621Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203219352Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203236952Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203256133Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203266873Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.203292434Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b1ffc0dc-c34d-4ce7-917c-ca0c1319b154","remote_addr":"[fd00:1122:3344:101::c]:58020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":468,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.245708463Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.245849547Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3c44cd0a-5c6a-42af-b995-239877d491ab","remote_addr":"[fd00:1122:3344:101::b]:60283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.291034159Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.29106481Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e101e9f4-8ee5-4c77-8666-4ef9d42104e7","remote_addr":"[fd00:1122:3344:101::b]:60283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.302318494Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4a263033-102d-4ec7-a23d-aaa0abbf99fd","remote_addr":"[fd00:1122:3344:101::b]:60283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.318980009Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49665"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.319065301Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.360184312Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.423344327Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dfc0f7b3-f3a0-4f13-a50e-974036bec3c4","remote_addr":"[fd00:1122:3344:101::b]:49665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104301,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.423529502Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.42378834Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e326e1e9-79dd-4c94-8db1-f3bf25bb13b2","remote_addr":"[fd00:1122:3344:101::b]:56096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":201,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.423822931Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a684f0b1-ca99-4af0-878a-b8eae2fa11ea","remote_addr":"[fd00:1122:3344:101::b]:49665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":238,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507078423Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55851"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507335351Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507392822Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507451914Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507483275Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507508496Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507534797Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507559637Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:26.507592718Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f4a22c6b-1239-432e-aeee-1a52bc248f11","remote_addr":"[fd00:1122:3344:101::b]:55851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":443,"response_code":"204"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:33.310178303Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:33.310264085Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:33.341919295Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999634s","next_cleanup":"Instant { tv_sec: 1936, tv_nsec: 684053216 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:47.444479452Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:54.600366555Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:54.600486699Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"699f72a9-8adb-457e-ad7b-31d7c02db08a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54993","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.302001206Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62378"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.302096729Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6455e52f-529d-4737-9ab4-0df688d4b422","remote_addr":"[fd00:1122:3344:101::a]:62378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.349726673Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.349759114Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1a5a2115-60d1-43c7-b55f-e507b91dce1e","remote_addr":"[fd00:1122:3344:101::a]:62378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.359215355Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"92798d06-554b-4334-9b1d-9856e35d6ad3","remote_addr":"[fd00:1122:3344:101::a]:62378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.421332709Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.421526405Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"22072c7c-48c7-4644-85ae-a892321f6160","remote_addr":"[fd00:1122:3344:101::a]:47824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.48670566Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48652"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.486847894Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.531259873Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.59853621Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2f9b0913-e1ea-4eea-82c1-f44c95888379","remote_addr":"[fd00:1122:3344:101::a]:48652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111740,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.59887091Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"723f3821-0bac-49a0-9849-1ec84b44cd0f","remote_addr":"[fd00:1122:3344:101::a]:48652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.636866318Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.637014952Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"929a11ff-0dcb-4e85-9503-269c99d57a40","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.677739492Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.677760362Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e75f3d8e-2474-4dfb-9de8-19cc40264b49","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.685994527Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"32680b88-46d3-46b3-bb1b-970c9ad3d856","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":270,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.869501385Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.869716551Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a056834e-2448-4378-94f2-50a1a8079bb0","remote_addr":"[fd00:1122:3344:101::c]:60492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.871674379Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55745"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.871804333Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.915239683Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.981785119Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"4c78fd34-722a-46c6-a781-713ba632a5b9","remote_addr":"[fd00:1122:3344:101::c]:55745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110030,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:55.982101818Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ef8cef64-df30-4574-b7a3-ab8ff0cc0b60","remote_addr":"[fd00:1122:3344:101::c]:55745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.026802975Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.0269484Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"83bc3ab5-7705-43f4-8a1b-a75f05dba5b8","remote_addr":"[fd00:1122:3344:101::b]:49283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.03977813Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.039799631Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"79c7a911-e4a7-4f10-b9cd-475616e2f47d","remote_addr":"[fd00:1122:3344:101::b]:49283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.049495499Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"52445268-8dd3-4b5e-90b1-aeb6b87e6d34","remote_addr":"[fd00:1122:3344:101::b]:49283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.05894575Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.059103174Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8bc81cfd-d784-4b01-977c-ee79acf1f6b5","remote_addr":"[fd00:1122:3344:101::b]:50177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.180496578Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40905"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.180591531Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.223518746Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.292215645Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"335d019c-2b45-44df-bcab-704b12b4da9c","remote_addr":"[fd00:1122:3344:101::b]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111661,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:24:56.292548945Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0f474142-901a-499e-afe9-a20c7084c69a","remote_addr":"[fd00:1122:3344:101::b]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:24.634659084Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:24.634766207Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3747a6d7-1d4c-49a0-a183-8626d4792485","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34984","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.393302098Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.393442492Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"95470577-fcc8-4ca4-832f-6ca1620c4ab6","remote_addr":"[fd00:1122:3344:101::a]:42995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":74,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.442712665Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.442750986Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"717ff63e-8157-44f5-8050-5be4fa53bb13","remote_addr":"[fd00:1122:3344:101::a]:42995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.453770063Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bec595b9-90c9-4645-8955-35a1e1180746","remote_addr":"[fd00:1122:3344:101::a]:42995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.605553269Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59335"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.605744055Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2b8d0c92-5602-4d0f-93dd-7dbdaf8ce4de","remote_addr":"[fd00:1122:3344:101::a]:59335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.726366056Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52558"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.726474669Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.766212149Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.835704242Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"265f33c4-be74-4f97-b07d-6283a0b02346","remote_addr":"[fd00:1122:3344:101::a]:52558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109263,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.835875667Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.836021882Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e80fec77-e55e-4eff-977d-0ceca788d8de","remote_addr":"[fd00:1122:3344:101::c]:64499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.836263159Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4b1f0205-c4f6-4602-bdc7-69fd5516c3f0","remote_addr":"[fd00:1122:3344:101::a]:52558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.864218819Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.86425038Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e1899920-9fa2-4830-9c4d-81945b8e7b10","remote_addr":"[fd00:1122:3344:101::c]:64499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.875813203Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ac087ab3-6af6-4ee2-a5d9-04409fb9805b","remote_addr":"[fd00:1122:3344:101::c]:64499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":265,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.959622301Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55943"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.959822147Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"acd3d496-39d9-4370-b26d-a30b4e9652a6","remote_addr":"[fd00:1122:3344:101::c]:55943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.975298147Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56202"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:25.97538949Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.018957833Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.078174841Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e489723a-06d0-495a-af9d-5f3f84d23c00","remote_addr":"[fd00:1122:3344:101::c]:56202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102810,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.081107928Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b57e4428-8e59-4996-9844-03bc610f120b","remote_addr":"[fd00:1122:3344:101::c]:56202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.111057707Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.111348876Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4ed7d8a8-ac00-4798-84bc-c9c38765c9c0","remote_addr":"[fd00:1122:3344:101::b]:47904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.114436718Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.114579402Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1ccc211c-f0ed-4847-8daf-ab6437415bd3","remote_addr":"[fd00:1122:3344:101::b]:44700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.16233288Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.162396752Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"33389a5d-e3d9-4aa7-a442-4b9276e5086b","remote_addr":"[fd00:1122:3344:101::b]:44700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.172640806Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5e8c316f-9039-4dca-bef8-85a3f11a7e7f","remote_addr":"[fd00:1122:3344:101::b]:44700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":259,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.362612606Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62225"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.362716119Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.402269223Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.460514003Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a65c522e-7f05-426a-952f-2a70e1f844c0","remote_addr":"[fd00:1122:3344:101::b]:62225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97846,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:26.460795251Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"13763291-3bbc-4244-99b7-ff139c3adde2","remote_addr":"[fd00:1122:3344:101::b]:62225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:54.667436399Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56085"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:54.667555892Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"38955f92-e467-4448-acc5-7bbc150af8da","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56085","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.419207809Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.419388734Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"381debb2-0123-497d-aacc-f251555fecaa","remote_addr":"[fd00:1122:3344:101::a]:44000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.43473542Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.434752281Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0700f9a0-1917-46a2-a4d9-5c760efa9692","remote_addr":"[fd00:1122:3344:101::a]:44000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.44618566Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.446385656Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"96a38db1-42f2-41e5-92c9-d175893807e0","remote_addr":"[fd00:1122:3344:101::a]:63331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.44921744Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b424bb53-5280-4931-8edf-edb17f958755","remote_addr":"[fd00:1122:3344:101::a]:44000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.495979748Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40364"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.496085241Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.546782576Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.614752784Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5ba4cd86-d294-4fde-846c-45e17d72d783","remote_addr":"[fd00:1122:3344:101::a]:40364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118711,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.615074994Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2707b4f9-df26-4a68-a9a6-cec8ab9b41c3","remote_addr":"[fd00:1122:3344:101::a]:40364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.618485005Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.61865791Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2d783b3f-99cc-4d4b-b12f-4685d327c282","remote_addr":"[fd00:1122:3344:101::c]:56433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.668341835Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.668373396Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"969755ac-8ffa-472b-b697-f494402912a1","remote_addr":"[fd00:1122:3344:101::c]:56433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.677670672Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1675275c-3d88-46fd-9c30-700209ba4b06","remote_addr":"[fd00:1122:3344:101::c]:56433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":237,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.780159615Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.780336161Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a705ab97-a7bd-4434-980e-bc3eb87517be","remote_addr":"[fd00:1122:3344:101::c]:34313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.798703796Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46360"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.798808879Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.849842364Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.926034506Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ae9252db-ea18-4191-9833-af16fc98dfe0","remote_addr":"[fd00:1122:3344:101::c]:46360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127254,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:55.926444578Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d5a7d66a-324e-436d-bb46-f9caaec98e1b","remote_addr":"[fd00:1122:3344:101::c]:46360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.064804756Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.06491871Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b06468a1-8200-439c-a740-87e1ce1cd179","remote_addr":"[fd00:1122:3344:101::b]:64153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.106063591Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.106086302Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8d82e103-4f98-407b-86e9-2d7639c9116b","remote_addr":"[fd00:1122:3344:101::b]:64153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.183712937Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43118"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.183886202Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2be08ec2-350f-402d-8c3c-d98ac8d01ef8","remote_addr":"[fd00:1122:3344:101::b]:43118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.19124533Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"764911ed-0103-4a62-b455-773b9b57d4f4","remote_addr":"[fd00:1122:3344:101::b]:64153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.301874305Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46593"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.301983198Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.339832032Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.393965589Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"76f58fc1-eab4-457e-8e14-16999ff01cb8","remote_addr":"[fd00:1122:3344:101::b]:46593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92033,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:25:56.394218296Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2ce4d09b-bae5-478d-8c63-0a46476bad7b","remote_addr":"[fd00:1122:3344:101::b]:46593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:24.700961657Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:24.701103671Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2cfad31e-1714-427e-a1f4-dcbd5b8c846b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53761","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.413460791Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56494"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.413682328Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"9ef401a7-2eb6-41d2-a019-cfd03679a5db","remote_addr":"[fd00:1122:3344:101::a]:56494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.45718851Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.457268932Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4e8c302f-4cc2-425a-9deb-eb474350b368","remote_addr":"[fd00:1122:3344:101::a]:54818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.502743542Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.502764513Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d45abe88-7977-4afc-a2e4-a12dd97f2f28","remote_addr":"[fd00:1122:3344:101::a]:54818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.512741559Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3685b776-120a-47aa-8056-a3fd6845adb7","remote_addr":"[fd00:1122:3344:101::a]:54818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":289,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.645552002Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58168"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.645717936Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.689306561Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.750147187Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"aee87c6c-6ff3-4a33-a101-519d4da0967e","remote_addr":"[fd00:1122:3344:101::a]:58168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104506,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.750428616Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6c5c29dd-6737-40e5-b5a0-31cc71b437d9","remote_addr":"[fd00:1122:3344:101::a]:58168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.793292718Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60428"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.793457543Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f5c67734-dfc6-4823-9e80-d2665025bda4","remote_addr":"[fd00:1122:3344:101::c]:60428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.865572374Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.865606395Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ba469d91-bd0f-4497-915e-190f0f3cfd99","remote_addr":"[fd00:1122:3344:101::c]:60428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:25.876372735Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"57a956b5-5cc3-48bc-b773-ffe4369fd4d8","remote_addr":"[fd00:1122:3344:101::c]:60428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":418,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.046078853Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.046257788Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3a1c9405-cb26-4361-a142-5bb692757692","remote_addr":"[fd00:1122:3344:101::c]:59414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.048694471Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47684"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.048792014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.090258075Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.150161583Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3a3c52c5-92a2-4aa2-914e-3d6e21e9ccac","remote_addr":"[fd00:1122:3344:101::c]:47684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101401,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.150410471Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35798"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.150656038Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ce6dc052-afc5-4798-a9f8-da60c3aa2ce3","remote_addr":"[fd00:1122:3344:101::b]:35798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.15070296Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"834fb87c-402b-4ee9-8e23-94d2012787f6","remote_addr":"[fd00:1122:3344:101::c]:47684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":269,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.201957171Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.201988532Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c3435cdc-9cf8-4783-bea1-9af46e212950","remote_addr":"[fd00:1122:3344:101::b]:35798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.210315039Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fce73671-047a-4fc2-9e1b-e717ac51b73c","remote_addr":"[fd00:1122:3344:101::b]:35798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":265,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.378125841Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44126"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.378395489Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c7ff41cb-58ab-4aac-ac3d-83b1c1d02d58","remote_addr":"[fd00:1122:3344:101::b]:44126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.466183556Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62096"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.46633838Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.513698717Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.576612955Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3ad2db25-d9ca-46da-a664-8029403e2c4c","remote_addr":"[fd00:1122:3344:101::b]:62096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110364,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:26.576882043Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2d78b2ac-39af-4d43-bf81-2bef3a73b455","remote_addr":"[fd00:1122:3344:101::b]:62096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:54.734074602Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:54.734180726Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"35c02a19-c7d4-47a4-9069-6abe0ecc37ca","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36740","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.311079263Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.311218258Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fd771871-19eb-4468-9e27-3960780a9bf6","remote_addr":"[fd00:1122:3344:101::a]:58736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.354720539Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.35474589Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"076f5f06-2b41-432e-b3bb-400fb9144b10","remote_addr":"[fd00:1122:3344:101::a]:58736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.358821791Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.359061908Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a048f971-7626-4564-b921-88830ade73da","remote_addr":"[fd00:1122:3344:101::a]:45160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.365759137Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"55f68b25-0399-441a-9c03-f1f1089424a9","remote_addr":"[fd00:1122:3344:101::a]:58736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.419674378Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49433"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.419785851Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.464170408Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.537618909Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9bc73ab9-add2-4cb2-81c7-f6fe0d2376dc","remote_addr":"[fd00:1122:3344:101::a]:49433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117865,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.538011721Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bc3021d2-bf9c-4048-999a-3ece37a908d7","remote_addr":"[fd00:1122:3344:101::a]:49433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.739685638Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51553"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.739833032Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5471ce57-dfde-4874-b092-b14c9d60e50d","remote_addr":"[fd00:1122:3344:101::c]:51553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.782777387Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.782802518Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4da138a6-e21b-4de3-812d-d8702cc9c29f","remote_addr":"[fd00:1122:3344:101::c]:51553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.792510106Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0816ab9a-5503-4442-996f-f27d8f644140","remote_addr":"[fd00:1122:3344:101::c]:51553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":271,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.914290882Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.914476817Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2ea461d7-1fa0-4e9f-9ff2-3e8a3408cad1","remote_addr":"[fd00:1122:3344:101::c]:59708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.94487322Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57183"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.944994883Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:55.991575026Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.055200895Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"370c676c-dbbe-4464-a675-73452a398e91","remote_addr":"[fd00:1122:3344:101::c]:57183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110262,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.05533603Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58641"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.055442933Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5777090a-3e4b-408e-a994-e1893130b469","remote_addr":"[fd00:1122:3344:101::b]:58641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.055590837Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"30dc78b1-885e-40d4-9be0-4707dc1ac6f9","remote_addr":"[fd00:1122:3344:101::c]:57183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.101768478Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.101805239Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"86ba43fb-2785-4e28-bfeb-8df0bebabf59","remote_addr":"[fd00:1122:3344:101::b]:58641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.122307428Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.122508154Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8fc63bab-fe79-45ab-bddf-91c6315a031a","remote_addr":"[fd00:1122:3344:101::b]:62800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.125690528Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5b3bc5c8-6756-4c05-bbc7-89a94dacd0f6","remote_addr":"[fd00:1122:3344:101::b]:58641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.191541423Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33202"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.191615326Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.234446517Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.294955094Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b0c230cc-bddf-4c44-a21b-3442d8e2abf7","remote_addr":"[fd00:1122:3344:101::b]:33202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103365,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:26:56.295353516Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"596ed3a8-681f-466a-a94c-c7f7e944f8ba","remote_addr":"[fd00:1122:3344:101::b]:33202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:24.767317239Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63327"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:24.767429743Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f6efe6c4-9d99-4e2f-bac7-53a2ba08a553","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63327","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.379283588Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.379433253Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"026d459b-d586-4f90-866c-5466b609e500","remote_addr":"[fd00:1122:3344:101::a]:39121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.427744087Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.427772088Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e73ae263-3127-4b4d-8f26-5c71ce05953e","remote_addr":"[fd00:1122:3344:101::a]:39121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.488412488Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.488615454Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"95597d27-26f0-41cc-8dd6-b6bb3bb18400","remote_addr":"[fd00:1122:3344:101::a]:45179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.496327234Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"951cebba-3bf3-480a-b2cf-31c8901785f9","remote_addr":"[fd00:1122:3344:101::a]:39121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.655144969Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52787"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.655263002Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.69695649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.764277049Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e3afac83-4735-4ada-843a-fae345d9c118","remote_addr":"[fd00:1122:3344:101::a]:52787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109052,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.764405512Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.764531306Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d21ca19e-b5f0-4083-a1de-3646cf83b280","remote_addr":"[fd00:1122:3344:101::c]:48095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.764703851Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3e263260-3f26-438e-a826-aec186001896","remote_addr":"[fd00:1122:3344:101::a]:52787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.811834161Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.811883282Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f01c6e1c-daaf-49bd-b1ed-e8e97963a998","remote_addr":"[fd00:1122:3344:101::c]:48095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.822798136Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"15f7924a-48b7-42ba-9ea5-82f40f526a1a","remote_addr":"[fd00:1122:3344:101::c]:48095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":306,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.828213987Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.828405993Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"642c60e0-7d25-4254-8d40-631948ed60ff","remote_addr":"[fd00:1122:3344:101::c]:39198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.995608087Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48174"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:25.995759581Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.04085275Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.101921383Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"97ad546c-27a9-473d-9e14-2a85670b9b96","remote_addr":"[fd00:1122:3344:101::c]:48174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106243,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.102199781Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1e99c064-8fea-4af9-b72c-ca1e6adf62ae","remote_addr":"[fd00:1122:3344:101::c]:48174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.113213048Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61471"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.113406884Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"969b9e33-613d-4dc8-ba4a-f2121089f5ec","remote_addr":"[fd00:1122:3344:101::b]:61471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.22879169Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.228934504Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5672f394-eb13-4925-8209-ed0b11535067","remote_addr":"[fd00:1122:3344:101::b]:64986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.272714944Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.272738094Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2aba2e12-26cc-4474-9499-cae780855628","remote_addr":"[fd00:1122:3344:101::b]:64986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.281464954Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ff43008c-b391-4832-9521-d7ca814a4694","remote_addr":"[fd00:1122:3344:101::b]:64986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":161,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.304618441Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57373"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.304704104Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.343525596Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.401023313Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5c53193e-2986-4fbe-b5eb-a3ad1e3279c6","remote_addr":"[fd00:1122:3344:101::b]:57373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96347,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:26.401281081Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"48462091-253c-4810-ba5e-1164255dc543","remote_addr":"[fd00:1122:3344:101::b]:57373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.467256168Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.468035311Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.468709772Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.46934645Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.47000652Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.47067207Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.471752482Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.475527614Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:28.476295157Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.464949884Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.465768148Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.466537641Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.467258492Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.467924242Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.468630613Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.469303073Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.469988153Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.470691284Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.473028533Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.473548579Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.474050154Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.474528788Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.474998992Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.475489977Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.476379853Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.476860457Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:38.478360562Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.468660137Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.469517613Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.470298586Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.471003086Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.471705537Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.472405118Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.473114369Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.47381145Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.47448267Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.476876511Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.477390176Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.477896481Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.478380876Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.478897481Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.479410736Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.480325733Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.480820318Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:48.481666653Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:54.801219046Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:54.801401122Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"28ab3f92-29a8-4c4e-a267-026f848600f2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44768","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.306856349Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.306956602Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cc077d5d-3f68-45d0-8b29-4ffd1f4bead8","remote_addr":"[fd00:1122:3344:101::a]:55346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.326857743Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.326879623Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"732dd844-9593-4ecc-952a-bcc4dfc9cbaa","remote_addr":"[fd00:1122:3344:101::a]:55346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.334352875Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38962"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.334549041Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"d3153f22-9f20-4867-832a-d425d5be063f","remote_addr":"[fd00:1122:3344:101::a]:38962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.344440545Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9a6cf151-adff-4fd8-bcc1-04aecabbb2b5","remote_addr":"[fd00:1122:3344:101::a]:55346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.402718005Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:32998"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.402824898Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.445793644Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.503560019Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a74caa19-f98b-4d30-9780-b97209bf25af","remote_addr":"[fd00:1122:3344:101::a]:32998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100780,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.503872758Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5231acc2-920f-4b1f-b751-77b542cbb1c2","remote_addr":"[fd00:1122:3344:101::a]:32998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.69188726Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.692022704Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"164434a6-e1e2-4dfb-a7f4-5acfcfd563e9","remote_addr":"[fd00:1122:3344:101::c]:36178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.735721531Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.735760672Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ea2c1adb-b2a3-484f-927d-3ebc4b939047","remote_addr":"[fd00:1122:3344:101::c]:36178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.744295956Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1cc85539-8196-4e77-b2e5-f8a80fd2e9af","remote_addr":"[fd00:1122:3344:101::c]:36178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":195,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.762362502Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43087"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.762465865Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.800530635Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.881942863Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ac048956-a358-411f-bd33-ef63c2a1599b","remote_addr":"[fd00:1122:3344:101::c]:43087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119488,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.882095727Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.882345304Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"48b3c0c4-0b30-42c0-b5de-4c064c411517","remote_addr":"[fd00:1122:3344:101::c]:51273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:55.88252342Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"950445d2-5825-4e80-b2cb-d0367aa352dc","remote_addr":"[fd00:1122:3344:101::c]:43087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.00714543Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.007284484Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8ce90d48-c349-428c-9254-89ec2eb9eff2","remote_addr":"[fd00:1122:3344:101::b]:64177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.046376714Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.046398945Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"53083c16-5c13-4337-b022-827cb385eff3","remote_addr":"[fd00:1122:3344:101::b]:64177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.054960809Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2e0c485e-3bf8-4223-a87d-59e239e48783","remote_addr":"[fd00:1122:3344:101::b]:64177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.089045211Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52536"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.089146324Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.130684828Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.192941206Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"726635d8-1d7a-4582-be96-0db4bdbd7b73","remote_addr":"[fd00:1122:3344:101::b]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103842,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.193215594Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35577"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.193467512Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"dbe2abe9-ad55-4127-8c23-b2a5549d3739","remote_addr":"[fd00:1122:3344:101::b]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":242,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:27:56.193595965Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"58d31281-a954-4e9e-a7ba-cd363edfcd67","remote_addr":"[fd00:1122:3344:101::b]:35577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":256,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.471634739Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.472398721Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.473081012Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.473839694Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.474504684Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.475173074Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.475853344Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.476518594Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.477200274Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.479520373Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.480047319Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.480535843Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.481017708Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.481534663Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.482014007Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.482901493Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.483385558Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:27:58.483874152Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.474395404Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.475197808Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.475922679Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.476695872Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.477399003Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.478147625Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.478834926Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.479544587Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.480238098Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.482639979Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.483196215Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.48370649Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.484199595Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.484670119Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.485141003Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.48605763Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.486526284Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:08.486992508Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.477860281Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.478633004Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.479317614Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.480033115Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.480692094Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.481400276Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.482083476Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.482745935Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.483456687Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.485742584Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.48627852Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.486809386Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.487317391Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.487797596Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.48827822Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.489147456Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.48964143Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:18.490157786Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:24.834107133Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:24.83433757Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"278dc9f5-f81c-475a-bea1-688fc749fbb0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47349","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.513647928Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.513807673Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"67fc0b49-9e8b-40c1-a91d-4c2989686445","remote_addr":"[fd00:1122:3344:101::a]:43235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.592562241Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.592606903Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b643b89f-e17e-4593-935b-bf9687713fcd","remote_addr":"[fd00:1122:3344:101::a]:43235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.671734161Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.672059101Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"674993f9-f780-4f08-93de-2d400c314475","remote_addr":"[fd00:1122:3344:101::a]:47349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.679159292Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f2220526-0565-4ad8-8f80-bb96d672667f","remote_addr":"[fd00:1122:3344:101::a]:43235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.700481875Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42043"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.700594038Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.743160102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.807112471Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cf3ffb29-16fc-4d2b-8ac4-f1fe821a181b","remote_addr":"[fd00:1122:3344:101::a]:42043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106543,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.807493902Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"38b8c27c-6356-4c4b-b8b1-b016045cbbb3","remote_addr":"[fd00:1122:3344:101::a]:42043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.993943938Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:25.994100262Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3192215e-876f-42fe-9cc3-4bc7c60e000b","remote_addr":"[fd00:1122:3344:101::c]:57339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.017088295Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.017301701Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"addddb53-cd54-4ac0-8150-cd6156e47cff","remote_addr":"[fd00:1122:3344:101::c]:50666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":151,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.043822499Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.043841179Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5fbe7864-917f-42d7-8631-169adc55ad09","remote_addr":"[fd00:1122:3344:101::c]:57339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.053777064Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ef107f34-d956-4a2c-ae31-07b563edef48","remote_addr":"[fd00:1122:3344:101::c]:57339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":281,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.090317029Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61795"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.090449743Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.140766187Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.223188424Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8fcd7ebd-5898-4c34-bfa4-47e8f97af1d8","remote_addr":"[fd00:1122:3344:101::c]:61795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132784,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.223462992Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.22373651Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7d01f75f-6b8d-4a45-82ae-0f3d07e1def9","remote_addr":"[fd00:1122:3344:101::c]:61795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":248,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.223745871Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5537bbba-8fc5-4e11-9ddd-d21d04798e9d","remote_addr":"[fd00:1122:3344:101::b]:53587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.245894198Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.245921169Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bd92b141-6cf5-4d0c-801e-130c04764dee","remote_addr":"[fd00:1122:3344:101::b]:53587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.255679568Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"41caaa6c-8371-4746-a5b6-3ea4f1b0cdaf","remote_addr":"[fd00:1122:3344:101::b]:53587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":259,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.29209247Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54666"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.292215183Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.336502358Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.399313663Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f3aed838-34e4-418e-9f4e-e14646c4f321","remote_addr":"[fd00:1122:3344:101::b]:54666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107142,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.39954789Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.399822068Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1a281c15-8dd6-4643-be20-dfb8c9e0ba50","remote_addr":"[fd00:1122:3344:101::b]:60242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:26.399837508Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"355e3551-a6f0-4f20-978e-8952d82a8569","remote_addr":"[fd00:1122:3344:101::b]:54666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":257,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.481905594Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.482724448Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.483417189Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.48413372Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.48481079Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.485517121Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.486194081Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.486879062Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.487596433Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.489952803Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.490437057Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.490945202Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.491416576Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.4918766Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.492350784Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.49322915Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.493705144Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:28.494180008Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.484095742Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.484918627Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.485661549Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.48637168Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.487073371Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.487768162Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.488496783Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.489175553Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.489870564Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.492208193Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.492726909Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.493275435Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.49378155Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.494279805Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.494749229Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.495639925Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.49612553Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:38.496596364Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.486502727Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.487311901Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.488041563Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.488776865Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.489471205Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.490182056Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.490872077Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.491554287Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.492248738Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.494567327Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.495068872Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.495586887Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.496111783Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.496599537Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.497091021Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.498014779Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.498506034Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:48.499002968Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:54.86769057Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52602"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:54.867780893Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5b1330c6-607d-4fb0-bcff-688424b18a48","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52602","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.322437951Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.322595896Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a42fd40e-ae56-4cbb-93ca-98cd08353534","remote_addr":"[fd00:1122:3344:101::a]:58942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.336973513Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.337009354Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"90538c72-94bf-4322-9014-b127b803d4d4","remote_addr":"[fd00:1122:3344:101::a]:58942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.346037462Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5965b3d5-3f29-458c-b9d5-ed479e6ae656","remote_addr":"[fd00:1122:3344:101::a]:58942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.444370241Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.444543687Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"6b64d4ec-a858-4d6a-ac27-16f7647b93d6","remote_addr":"[fd00:1122:3344:101::a]:40940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.465906041Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48940"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.465996063Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.511378451Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.5955534Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"84fbeaa5-107d-4881-bec2-0b260cd5687f","remote_addr":"[fd00:1122:3344:101::a]:48940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129577,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.595934481Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5d255e83-45b3-42f8-a106-2ff2c2a6ee29","remote_addr":"[fd00:1122:3344:101::a]:48940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.728037323Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.728167327Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"136bdbe1-771c-4058-b673-12c3569609e4","remote_addr":"[fd00:1122:3344:101::c]:64363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.76598055Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.766169535Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"abc4a096-a65c-461a-a458-a58492847881","remote_addr":"[fd00:1122:3344:101::c]:43936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.779434359Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.77945692Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1c5371be-40a0-4792-b948-4111d7fe3b8b","remote_addr":"[fd00:1122:3344:101::c]:64363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.788074526Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"98df2d82-a30e-42bd-8a38-32bf150ceeeb","remote_addr":"[fd00:1122:3344:101::c]:64363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.908993256Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.909084738Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"583b93a3-8788-447c-a1d2-db64eab52518","remote_addr":"[fd00:1122:3344:101::b]:56158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.953201418Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34780"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.953325532Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:55.999282246Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.063592026Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f2d9d76c-8c41-4005-892a-afac4fe2f2b1","remote_addr":"[fd00:1122:3344:101::c]:34780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110318,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.063925655Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.063962517Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"dfcd3006-6054-4111-b6f5-3550244a330d","remote_addr":"[fd00:1122:3344:101::b]:56158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.06408733Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"69b05060-8e81-4b2e-a522-8ae1e31ef18c","remote_addr":"[fd00:1122:3344:101::c]:34780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.074939533Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.075213171Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"629061e6-0f24-48cb-951f-56b032f46eee","remote_addr":"[fd00:1122:3344:101::b]:48367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.077500029Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"301984f5-afc5-4eaf-a70e-063ed8437c9e","remote_addr":"[fd00:1122:3344:101::b]:56158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.106200111Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48842"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.106278353Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.145429635Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.213091034Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ed08ad01-1717-4667-85c3-170173b061b5","remote_addr":"[fd00:1122:3344:101::b]:48842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106826,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:28:56.213408444Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a306ed4f-db91-4b28-89fc-c56688a28b78","remote_addr":"[fd00:1122:3344:101::b]:48842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.490305313Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.491128428Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.491833229Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.492528629Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.493196429Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.4939087Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.49457865Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.49523732Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.49591Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.498214198Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.498722713Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.499252349Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.499766254Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.500256109Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.500729553Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.501600399Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.502068453Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:28:58.502529766Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.493996836Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.49480486Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.495497191Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.496230682Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.496916043Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.497607443Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.498309504Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.498986574Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.499669445Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.501969713Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.502505339Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.503025434Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.503522079Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.504049765Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.504531989Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.505447356Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.505943981Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:08.506436305Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.498146133Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.498919396Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.499646177Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.500335868Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.501036269Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.50174223Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.50241455Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.50311043Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.503787271Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.506027717Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.506530132Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.507034837Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.507516041Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.507971865Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.508437189Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.509291744Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.509775928Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:18.510287584Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:24.900807583Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:24.900956448Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b457eb00-7e29-4cc3-ba79-d8505369cf1d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51414","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.451417431Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39812"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.451585596Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f78cff82-c9f5-4528-81a0-cac52d723a95","remote_addr":"[fd00:1122:3344:101::a]:39812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.535668542Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.535723564Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"009dd5c6-6567-4467-923d-0e846c19eca1","remote_addr":"[fd00:1122:3344:101::a]:39812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.548368459Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9a6eb4f0-3d7a-4ecd-a6f7-abdbd3385f19","remote_addr":"[fd00:1122:3344:101::a]:39812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":306,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.813792769Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.813904112Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e1ec9a01-9026-4f69-8722-9d6f3e3b4ab9","remote_addr":"[fd00:1122:3344:101::c]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.816463768Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46821"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.816675925Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"62a9efbd-64a5-490e-89e2-1539fda03de3","remote_addr":"[fd00:1122:3344:101::a]:46821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.841566174Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45892"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.841888263Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.904648827Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989328141Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ab1c43ca-feb9-45bb-8ae1-f63a036a4b67","remote_addr":"[fd00:1122:3344:101::a]:45892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147507,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989567238Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989717092Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1eac5a9b-b4cc-491c-9026-eb08c7d16296","remote_addr":"[fd00:1122:3344:101::a]:45892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989770884Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989779704Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.989836126Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:25.9902954Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.013996913Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55435"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014329413Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014376604Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014407765Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014425756Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014438206Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014455917Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014461447Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.014493888Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e91ab1a8-e106-427a-bd7b-81f65266eb65","remote_addr":"[fd00:1122:3344:101::a]:55435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":369,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.080965912Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.081352103Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"72069104-4f5c-4379-9d39-53a85e1159bb","remote_addr":"[fd00:1122:3344:101::c]:37415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":229,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.129991407Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.130528483Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9407faa7-a722-48ab-8bff-01b82fc3a516","remote_addr":"[fd00:1122:3344:101::b]:42188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":254,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.130638186Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.130702308Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"51af2653-860e-4c1b-9f83-737c10f7eb31","remote_addr":"[fd00:1122:3344:101::c]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":279,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.18836305Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40360"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.188567616Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.188581976Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.18869005Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.195994237Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.19609317Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2faf98d5-7ae0-47e1-8598-41bfe5d0b5be","remote_addr":"[fd00:1122:3344:101::b]:42188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":830,"response_code":"204"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.196360858Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.204393436Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fbf173b9-6aa6-41ad-b726-bb12404b1720","remote_addr":"[fd00:1122:3344:101::c]:54306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1196,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.220505874Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b7184525-625d-4e1e-910c-9a7b70eaa113","remote_addr":"[fd00:1122:3344:101::b]:42188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":13483,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.259518723Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60295"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.259656237Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.266330985Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"f03addf3-ecd4-4fd3-a445-2f8581b3953d","remote_addr":"[fd00:1122:3344:101::a]:38708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":276587,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.306293141Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324250275Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46176"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324716348Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324788831Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324842832Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324875313Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324906194Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324947385Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324963796Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.324974216Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8595c2fd-203d-4ef3-a69b-7c5ef4c3b74a","remote_addr":"[fd00:1122:3344:101::c]:46176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":558,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.372675192Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e835755d-0dcb-4836-88eb-e620fd8f682f","remote_addr":"[fd00:1122:3344:101::c]:60295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113055,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.373058564Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f8d83594-1006-4ca6-8bef-db530bb4cbd8","remote_addr":"[fd00:1122:3344:101::c]:60295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.376505646Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44658"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.376618109Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.37662659Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.376671441Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.376965589Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.469620151Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"bf5b40b7-448b-4690-a74d-b8f821a02df8","remote_addr":"[fd00:1122:3344:101::c]:40360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":281116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.478162424Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39523"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.4786882Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"42e035f5-18ce-475c-bb03-2874e31c7d16","remote_addr":"[fd00:1122:3344:101::b]:39523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":307,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563550589Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49137"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563896939Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5bdcd428-292d-4d10-9115-282b4950a6c7, kind: Service { id: 309c9aef-da72-4f08-9f70-c20d80316472 }, name: Name(\"external-dns-309c9aef-da72-4f08-9f70-c20d80316472\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 162])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7bddae3f-1834-4c3e-96a7-d7374171c136, kind: Service { id: 9bf7f7e3-0a6c-48b6-84be-c4bb8425f761 }, name: Name(\"external-dns-9bf7f7e3-0a6c-48b6-84be-c4bb8425f761\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 166, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bbd7b6-d736-4142-9ad8-60e594545f3d, kind: Service { id: f13e9b98-ae91-4d61-ba9b-3b178cfa151a }, name: Name(\"nexus-f13e9b98-ae91-4d61-ba9b-3b178cfa151a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: acdf8cb7-85f2-4777-8bd4-f403275388d3, kind: Service { id: f633068a-05f8-4be2-a65b-d83ac560f7a9 }, name: Name(\"nexus-f633068a-05f8-4be2-a65b-d83ac560f7a9\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 111])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b6096ff9-4d2b-4b07-af5b-bb3cff616f69, kind: Service { id: e2378399-e8ed-453c-847a-0a419dfd0966 }, name: Name(\"nexus-e2378399-e8ed-453c-847a-0a419dfd0966\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563933671Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563966472Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563978522Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.563993282Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.564006883Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.564015833Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.564039254Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9c84c94e-5aff-40a1-98e0-15aad39e242f","remote_addr":"[fd00:1122:3344:101::b]:49137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":353,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.59220259Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49872"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.592712475Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.630915569Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"4e685253-bc66-4f33-9c17-82910199b906","remote_addr":"[fd00:1122:3344:101::b]:44658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":254312,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.636884026Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.708569934Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9730ba89-afba-486d-8f17-0655da13aaf9","remote_addr":"[fd00:1122:3344:101::b]:49872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116082,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:26.708972937Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0142cd9f-c707-4eb5-aa50-f7484da645a7","remote_addr":"[fd00:1122:3344:101::b]:49872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.502380902Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.503201896Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.503918288Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.50466749Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.505376241Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.506089662Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.506801063Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.507479963Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.508171104Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.510537324Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.51108575Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.511615096Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.51210576Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.512609186Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.5131013Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.514015067Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.514517232Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:28.515041038Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.505704544Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.506531208Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.50725246Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.507967901Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.508644401Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.509353742Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.510063283Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.510721943Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.511410923Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.513742273Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.514257268Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.514776583Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.515275028Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.515787903Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.516261467Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.517133413Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.517619837Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:38.518121412Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:47.44209916Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.508235572Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.509039026Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.509781288Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.51052512Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.511221721Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.511899541Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.512568161Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.513259621Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.513948702Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.516303512Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.516809087Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.517300352Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.517793576Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.518307311Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.518785856Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.519653441Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.520124915Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:48.5206318Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:54.933420031Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45529"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:54.933558866Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"deca7a28-9d94-4636-abc9-83eab89369ad","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45529","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.298164261Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49222"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.298297475Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8a02d4c9-c9e9-42c4-8879-53c4d4073a2f","remote_addr":"[fd00:1122:3344:101::a]:49222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.313241238Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.313256599Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f34cc326-c7e5-4b02-93b6-640ed423e8aa","remote_addr":"[fd00:1122:3344:101::a]:49222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.32204438Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"741c8625-3f24-4bdc-a0c5-d384cbc4b495","remote_addr":"[fd00:1122:3344:101::a]:49222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.447318219Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.447496314Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"41e57af6-0598-45a3-ac56-69afab2c4b3a","remote_addr":"[fd00:1122:3344:101::a]:46470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.495743077Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55376"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.495833869Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.549633067Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.616200573Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7b8a5ae0-bb1c-4a5a-8f7c-9a69477f55c5","remote_addr":"[fd00:1122:3344:101::a]:55376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120396,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.616543003Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a59ef854-881c-42ab-9892-575d88abe4fe","remote_addr":"[fd00:1122:3344:101::a]:55376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.63363342Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.633787485Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"79cc3ea5-b166-4bd6-9ba5-e190838b01c2","remote_addr":"[fd00:1122:3344:101::c]:61585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.680719238Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.680744429Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0d04b49b-4623-4e07-bf0c-4f4b71c08f41","remote_addr":"[fd00:1122:3344:101::c]:61585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.690751176Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"02bb985c-0df9-44d6-8e8d-7e125425e655","remote_addr":"[fd00:1122:3344:101::c]:61585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.730560468Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.730710362Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"26dbf8f7-e181-4805-a6ab-1bb89a6d54a4","remote_addr":"[fd00:1122:3344:101::c]:36034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.79293844Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59295"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.793041993Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.841266405Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.908583014Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"54fbefe4-7235-4bfe-841e-ccc25c7a5abf","remote_addr":"[fd00:1122:3344:101::c]:59295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115571,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:55.908919873Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9d4a52c9-bfcf-4b82-979d-33e384ce0031","remote_addr":"[fd00:1122:3344:101::c]:59295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.043778508Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60849"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.043956283Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"159f8495-8594-455f-98f3-5c49b6ba4cc9","remote_addr":"[fd00:1122:3344:101::b]:60849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.084105995Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.084126255Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9b7ad341-7740-4986-b61d-acff2e9272e7","remote_addr":"[fd00:1122:3344:101::b]:60849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.094839933Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.095017189Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"933ffbc2-935f-490b-8068-315b07161fac","remote_addr":"[fd00:1122:3344:101::b]:52137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.103279204Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a4bfbc31-9eb4-4c7b-9d54-478f2a16d445","remote_addr":"[fd00:1122:3344:101::b]:60849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.141353974Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50173"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.141456817Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.179469686Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.242459316Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"79f817b7-b1b3-4346-84c6-e514da219c2e","remote_addr":"[fd00:1122:3344:101::b]:50173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101012,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:29:56.242787996Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"86207fd7-dd26-4cf7-99ea-b27aed01a76b","remote_addr":"[fd00:1122:3344:101::b]:50173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.50975217Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.510542944Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.511269366Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.513010017Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.513695167Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.514395238Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.515077408Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.515737708Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.516436279Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.518728067Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.519212731Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.519706406Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.52018557Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.520655384Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.521112368Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.522003964Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.522475658Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:29:58.522959173Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.511870603Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.512599875Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.513307406Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.514042538Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.515717358Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.516395778Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.517043837Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.517694026Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.518374956Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.520658624Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.52116549Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.521670765Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.522139488Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.522601082Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.523066716Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.523922231Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.524400346Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:08.52489119Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.513735931Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.514519434Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.515237765Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.515943176Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.517647647Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.518351988Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.519075869Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.519748259Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.520467491Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.522741848Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.523251803Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.523775909Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.524282864Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.524775209Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.525242932Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.526135729Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.526616803Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:18.527109278Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:24.967781847Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:24.967940951Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9e0ebb24-8dc0-4bab-b419-e98d1d5796dd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60614","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.414351005Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.414455288Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0544352a-9620-484e-8fbd-c50f5156cbc5","remote_addr":"[fd00:1122:3344:101::a]:61214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.468264516Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.468321427Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e7d3212a-54a4-4ff3-a313-9c1bdba01fb1","remote_addr":"[fd00:1122:3344:101::a]:61214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.479436348Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"da8645ab-f468-41fb-a901-ddf231fe2ae7","remote_addr":"[fd00:1122:3344:101::a]:61214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":271,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.598595805Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49263"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.598889904Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"29856f12-9216-4c61-b3dc-0e4fcd96e78d","remote_addr":"[fd00:1122:3344:101::a]:49263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":247,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.69814395Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50413"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.698249284Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.741309232Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.799624434Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"51314b78-af12-4f24-885a-9ca9c51dc740","remote_addr":"[fd00:1122:3344:101::a]:50413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101430,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.799935473Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ecda5b06-cb34-4b1e-bb7c-a3bbba3fecb4","remote_addr":"[fd00:1122:3344:101::a]:50413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.901780587Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.901953192Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8821a2b8-6972-408f-bbfb-46dcb4b43301","remote_addr":"[fd00:1122:3344:101::c]:35508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.930267142Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.930301033Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"77ad00d0-b920-4530-92b0-72a86742c83e","remote_addr":"[fd00:1122:3344:101::c]:35508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.94131785Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"20f6e358-ac81-4540-850d-5278a2e15449","remote_addr":"[fd00:1122:3344:101::c]:35508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":230,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.972265559Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.972466445Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"cf91979f-d3a4-4f0b-9151-36bb201ee890","remote_addr":"[fd00:1122:3344:101::c]:62252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.991267093Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61862"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:25.991358386Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.035346122Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.102195357Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e6c4df9b-9bd5-4b6a-98e1-7e385fb7cf6b","remote_addr":"[fd00:1122:3344:101::c]:61862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110870,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.102509396Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2f06049c-455f-45ab-a3e6-4750293cd8d0","remote_addr":"[fd00:1122:3344:101::c]:61862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.155972463Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47680"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.1561926Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2fa01419-0846-4223-83b7-a8091fb63222","remote_addr":"[fd00:1122:3344:101::b]:47680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.298472384Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.298618588Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"40850f54-f252-49bf-a17a-057d62973dc9","remote_addr":"[fd00:1122:3344:101::b]:60346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.340041578Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.340079259Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"04895987-9d8f-4db5-98a9-f340972dc0e6","remote_addr":"[fd00:1122:3344:101::b]:60346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.348990154Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9751df67-09a8-4ff0-9bf7-6d1e1a982d73","remote_addr":"[fd00:1122:3344:101::b]:60346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.395443433Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50068"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.395535466Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.438749869Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.495490263Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"afed8e99-29ac-4e17-aa77-c3138e5e2d32","remote_addr":"[fd00:1122:3344:101::b]:50068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99987,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:26.495841644Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cc2cdd7e-6e76-45f9-a50b-d452003f81c6","remote_addr":"[fd00:1122:3344:101::b]:50068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.515454515Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.516193657Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.516869347Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.517566408Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.519272358Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.519974609Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.520662999Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.52134462Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.52201736Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.524297317Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.524769351Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.525255816Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.525717019Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.526188303Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.526624386Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.527473391Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.527932075Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:28.528383469Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.517395336Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.518191259Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.51888471Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.519589401Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.521293551Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.521953181Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.522627121Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.52328107Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.523967661Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.526239428Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.526762374Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.527240938Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.527717052Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.528191926Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.52865649Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.529556466Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.530037241Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:38.530526195Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.519763659Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.520782779Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.521452379Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.522145079Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.522814339Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.523453898Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.524296973Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.524974514Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.525647843Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.527945282Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.528460007Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.528934171Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.529405985Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.529846518Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.530311022Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.531159237Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.53161452Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:48.532093695Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.000353953Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.000450946Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"274d22b0-35f6-426b-91ae-c55c44853aca","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35233","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.363904817Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.364039601Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7bce6acd-6a4a-4ddf-a3a4-eb6c23181e7f","remote_addr":"[fd00:1122:3344:101::a]:52464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.406768939Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.40679623Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2934cd79-a5c6-42a0-a41e-ad46ee5a62ef","remote_addr":"[fd00:1122:3344:101::a]:52464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.415662023Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"62a7191d-a41a-449b-bc14-20a258fc6719","remote_addr":"[fd00:1122:3344:101::a]:52464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.499304427Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.499508753Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"bab2811f-6453-4584-8d4d-dd2287a1909a","remote_addr":"[fd00:1122:3344:101::a]:49231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.550636751Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38913"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.550747044Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.593608957Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.670503029Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a884aaf5-2798-4448-8047-dbe49206553e","remote_addr":"[fd00:1122:3344:101::a]:38913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119796,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.670665314Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.670814568Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2118be66-4ff0-49b8-a4d8-cae1881d9baf","remote_addr":"[fd00:1122:3344:101::c]:48203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.670946462Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7fbf993a-5611-445f-be5a-c91087798aab","remote_addr":"[fd00:1122:3344:101::a]:38913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":209,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.711376113Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.711436664Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c235466a-10bf-42a7-8f51-ff93fd29ae05","remote_addr":"[fd00:1122:3344:101::c]:48203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.720841323Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a1c47c6f-9c3c-40b6-9fbf-d5d24747f34d","remote_addr":"[fd00:1122:3344:101::c]:48203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":194,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.752679529Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.752840084Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"04abf691-ba4b-4617-888a-710a7baab518","remote_addr":"[fd00:1122:3344:101::c]:60063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.758335457Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39437"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.758419219Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.80157247Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.892532421Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3bd3e079-7544-4dff-9baa-3ddc7d85c804","remote_addr":"[fd00:1122:3344:101::c]:39437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134133,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:55.89384262Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5fc13fa5-b99c-4926-b7ac-023eab1fbd1b","remote_addr":"[fd00:1122:3344:101::c]:39437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":998,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.028620461Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.028747775Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6ae75f49-14ca-4d0c-8509-5cb6071f06e7","remote_addr":"[fd00:1122:3344:101::b]:41560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.072246687Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.072276288Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5eb958ec-8a77-4a97-b280-8ff2a52c26eb","remote_addr":"[fd00:1122:3344:101::b]:41560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":121,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.082519041Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"10e7202f-5246-4055-86c0-c68eac353e3d","remote_addr":"[fd00:1122:3344:101::b]:41560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.130554988Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.130716752Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b6cd3d7c-4820-4ac5-ac46-a2fe6b2db7ab","remote_addr":"[fd00:1122:3344:101::b]:44365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.157268631Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53282"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.157368194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.200803653Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.262882167Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7f4217c5-328f-4053-8ba5-d850de588f21","remote_addr":"[fd00:1122:3344:101::b]:53282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105553,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:30:56.263277688Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6dd048e2-c643-444e-a805-61f532169aee","remote_addr":"[fd00:1122:3344:101::b]:53282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":159,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.522346579Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.523059Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.52374673Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.52441859Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.525062329Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.525720319Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.526386439Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.527036478Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.527694537Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.530002636Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.53047433Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.530951694Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.531414488Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.531862491Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.532303784Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.53315767Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.533626324Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:30:58.534069847Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.523998781Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.524789254Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.525469384Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.526172095Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.526831895Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.527479354Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.528134944Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.528784853Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.529438462Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.53172107Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.532211275Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.532685029Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.533148332Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.533603606Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.53405771Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.534941816Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.53541172Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:08.535872403Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:16.939648546Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52973"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:16.939920654Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"21745ed3-b851-4733-b289-cef5c0a1e2e2","remote_addr":"[fd00:1122:3344:101::a]:52973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":176,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:16.940200912Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(7291606) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:16.940240633Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"PUT","req_id":"af9a04c2-16de-4af0-a6b7-d6829e191348","remote_addr":"[fd00:1122:3344:101::a]:52973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.231359726Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35975"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.232653155Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 538efd2f-b8ed-4a95-91b1-680b63627575, project_id: 6ce0f376-6379-45db-a017-9b7dfee5a397 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-06T00:31:17.231134040Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(7291606))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-ef8427e1dc1\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ad0007b8-c89d-4eef-be3e-370ac4b3b228, read_only_parent: Some(Volume { block_size: 512, id: ecc108be-8e6e-4b05-84f4-e4363804ab4c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9c7195dc-564e-4802-9ddd-2c776c0243ac, key: Some(\"pEhCCKcajsWHYdGBfFZbyx006TU0Sb7Fi182emwFvGM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::1a]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ad0007b8-c89d-4eef-be3e-370ac4b3b228, key: Some(\"K5De4lNTygH/yc/q7UW5VQE6U4voiR+ocX1uzMc7hWw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-ef8427e1dc1\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.232686166Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.232691406Z","hostname":"sock","pid":1572,"propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(7291606))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d04c5890-48aa-4e4d-951b-e3f160bd3d6f, kind: Instance { id: c7df1963-d43c-46bf-b3d4-6a37670b026b }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(7291606), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-ef8427e1dc1\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ad0007b8-c89d-4eef-be3e-370ac4b3b228, read_only_parent: Some(Volume { block_size: 512, id: ecc108be-8e6e-4b05-84f4-e4363804ab4c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9c7195dc-564e-4802-9ddd-2c776c0243ac, key: Some(\"pEhCCKcajsWHYdGBfFZbyx006TU0Sb7Fi182emwFvGM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::1a]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ad0007b8-c89d-4eef-be3e-370ac4b3b228, key: Some(\"K5De4lNTygH/yc/q7UW5VQE6U4voiR+ocX1uzMc7hWw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-ef8427e1dc1\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-06T00:31:17.231134040Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.233442978Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524","method":"PUT","req_id":"b8a722e6-4766-4cfb-801e-077259b23c93","remote_addr":"[fd00:1122:3344:101::a]:35975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1994,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.308984901Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56298"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.30929698Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524/state","method":"PUT","req_id":"028abc81-0826-48a4-9217-14bfabd7b10d","remote_addr":"[fd00:1122:3344:101::a]:56298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.31064253Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 7291606 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.310681491Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.354521363Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.379488894Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.411470214Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43542"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.411653109Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5b439d9e-3895-46bc-8f2f-025a673ace85","remote_addr":"[fd00:1122:3344:101::a]:43542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:17.431988883Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {d04c5890-48aa-4e4d-951b-e3f160bd3d6f: {10.151.2.175: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}, 10.151.2.176: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}}} }"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.526275921Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.527277161Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.528226209Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.529198428Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.530116935Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.531031103Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.5319617Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.532848767Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.533786635Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.538497784Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.539201355Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.539895576Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.540602727Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.541259486Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.541942007Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.543572945Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.544275556Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.544938916Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"Profile for oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:18.646015806Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:18.646303365Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 30.257864ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.892795073Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 132.990107ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:18.941047615Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 194.215467ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:19.092257425Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 514.701077ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:19.300853888Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 1.178539552s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:19.832814812Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 572.011427ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:21.02612181Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 1.109132188s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:21.611877021Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 1.099506157s","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:22.734661405Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:23.868998593Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 27.124426ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:23.882247447Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 99.411735ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:23.92143219Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 255.418117ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:24.033953451Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\") failed: Property not found. retry in 468.143461ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:24.302479313Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.78403714Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.820600075Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.956400717Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.957278053Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.957340295Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5b4b335e-141e-4bbc-b6de-ec804a97f66d","remote_addr":"[fd00:1122:3344:101::a]:43542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7525524,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-11-06T00:31:24.957357886Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.957380076Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.957392457Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-06T00:31:24.957357886Z }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:24.957400527Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-06T00:31:24.957357886Z }, migration_in: None, migration_out: None }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-11-06T00:31:25.010128022Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.010173234Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.010245836Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-06T00:31:25.010128022Z }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.010258766Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-06T00:31:25.010128022Z }, migration_in: None, migration_out: None }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.035009291Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.035165096Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"affa7796-79e9-4d03-96bf-a8a0206c5bd9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45248","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.041113073Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7c2f4f79-2aca-4c58-a79f-fab5e44d285c","remote_addr":"[fd00:1122:3344:101::a]:43542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":520,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.329867095Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57122"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.329989079Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3bf6e831-2c8d-4bfe-92d1-9490eaf45d06","remote_addr":"[fd00:1122:3344:101::a]:57122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.389428294Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {d04c5890-48aa-4e4d-951b-e3f160bd3d6f: {10.151.2.175: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}, 10.151.2.176: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.389470445Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c29dcbe7-6e78-46a1-8714-35cf4ef16577","remote_addr":"[fd00:1122:3344:101::a]:57122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.407382657Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2de75c2e-e638-494b-861d-0d980cf2b70f","remote_addr":"[fd00:1122:3344:101::a]:57122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":318,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.476331684Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57454"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.476481118Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524/state","method":"GET","req_id":"087062b6-a315-4059-b936-d9fff0b44447","remote_addr":"[fd00:1122:3344:101::a]:57454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.543416816Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.543674253Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"1c0839fd-60b6-4ae1-aec4-71c4a20f3ce9","remote_addr":"[fd00:1122:3344:101::a]:55102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.627476701Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61553"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.627590325Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.671457747Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.734321813Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b9e178b0-b77e-4492-a431-e7149343252b","remote_addr":"[fd00:1122:3344:101::a]:61553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106787,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.734623802Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"34f4e1df-8094-4db1-a166-fb723d1778c6","remote_addr":"[fd00:1122:3344:101::a]:61553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.755798191Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48568"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.756031718Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524/state","method":"GET","req_id":"b6acd157-9cc2-4d89-b940-55b7121feb48","remote_addr":"[fd00:1122:3344:101::c]:48568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.971022791Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:25.971140954Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d9d6a087-cb91-4a44-8b10-48757fbe6570","remote_addr":"[fd00:1122:3344:101::c]:50343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.040011109Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.040297367Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"8aad9986-4fd7-49f3-9064-8ade7dbb0fe2","remote_addr":"[fd00:1122:3344:101::c]:50872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":202,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.084784518Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {d04c5890-48aa-4e4d-951b-e3f160bd3d6f: {10.151.2.175: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}, 10.151.2.176: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.08484962Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e7260eb2-213a-4618-8a00-d69e5c1e1502","remote_addr":"[fd00:1122:3344:101::c]:50343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.08786105Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63988"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.087973413Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.133492225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.200352999Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e0cc9c30-cf4a-46d3-b999-88309d11f570","remote_addr":"[fd00:1122:3344:101::c]:63988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112415,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.200644398Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.200945497Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"612ca547-c1f4-40e8-833d-cf8c82d1363a","remote_addr":"[fd00:1122:3344:101::c]:50343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":428,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.201003179Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4fbc4037-5c3e-4ee5-916d-d0e137f16f86","remote_addr":"[fd00:1122:3344:101::c]:63988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":330,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.201026659Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"920a0f6f-6d26-4d54-83af-5def88a6364c","remote_addr":"[fd00:1122:3344:101::b]:41111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.230035531Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.230214856Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524/state","method":"GET","req_id":"2668b29a-b5e9-413e-9a7a-99ba6ab2086c","remote_addr":"[fd00:1122:3344:101::b]:39472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.281983973Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {d04c5890-48aa-4e4d-951b-e3f160bd3d6f: {10.151.2.176: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}, 10.151.2.175: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.282022624Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"39d029c7-34ef-4ade-8773-e098bef4ed05","remote_addr":"[fd00:1122:3344:101::b]:41111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.352811326Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.353008762Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b5c8ef6e-4d3e-4385-9984-2d3e2ca7dbdc","remote_addr":"[fd00:1122:3344:101::b]:51991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.356067063Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9b36caa8-0292-4ca9-a488-32c2ecc7049f","remote_addr":"[fd00:1122:3344:101::b]:41111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.368603035Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64761"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.368688017Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.412417556Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.470721107Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0d59baa5-68f7-4e87-8add-e859587382d5","remote_addr":"[fd00:1122:3344:101::b]:64761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102064,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:26.471060867Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"dd45e977-4a29-44c6-bcc8-f03d34348abc","remote_addr":"[fd00:1122:3344:101::b]:64761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.527869612Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.528651105Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.529355606Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.531081897Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.531910942Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.532737847Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.533540121Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.535252521Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.535986673Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.539497978Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.541045773Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.541553348Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.542032493Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.542504477Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.543985341Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.544859227Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.545350511Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:28.546840905Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.856049001Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56253"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.856815314Z","hostname":"sock","pid":1572,"uri":"/vmms/46330988-8b9f-455b-aa6c-07ae8d3bc524/state","method":"PUT","req_id":"7b217a7c-af15-4bff-8ee2-e70f59469477","remote_addr":"[fd00:1122:3344:101::a]:56253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":590,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-11-06T00:31:34.856929597Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.856951288Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.85700745Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-06T00:31:34.856929597Z }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.85702489Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-06T00:31:34.856929597Z }, migration_in: None, migration_out: None }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-11-06T00:31:34.894441111Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.894469582Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.894499473Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-11-06T00:31:34.894441111Z }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.894507613Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.894517503Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:34.894601266Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"4590287824976790885"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:34.894638517Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"2625257786590237154"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.89474588Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:34.894786491Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/0a098d83-7312-4c1d-895e-4802f8c14d04/debug/bundle/zone\", \"/pool/int/a12a67dd-438b-47b6-9a37-99f4c825ac9d/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/f049eab6-1590-47c8-b087-bc798a2cfc73/crypt/debug\", \"/pool/ext/9a4fd05b-1fb2-49d1-ab40-b32093e6fd86/crypt/debug\", \"/pool/ext/25d63394-e252-4651-bd2a-cb7cb7ea5b3e/crypt/debug\", \"/pool/ext/aaf2d272-1feb-4e24-ba54-7075c6ab94d1/crypt/debug\", \"/pool/ext/b89e85b3-c4ff-4874-bdbd-5e8cca783de4/crypt/debug\", \"/pool/ext/a62f7ed5-f062-4f65-8692-31c405e42f70/crypt/debug\", \"/pool/ext/392a3437-26be-4bcb-8f19-18fd4492b7e0/crypt/debug\", \"/pool/ext/60f04be8-23a0-4089-b45c-357c38441897/crypt/debug\", \"/pool/ext/0ff26524-4a22-4458-bf83-380365ee3e3b/crypt/debug\"] }","zone_name":"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:35.547257583Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524\", bundle_id: e4181b2f-466a-459c-9abb-6669cf35eca1 }, time_created: 2024-11-06T00:31:34.895679778Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_46330988-8b9f-455b-aa6c-07ae8d3bc524","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:35.547342875Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.285432428Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.302953438Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(7291606), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.3030244Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(7291606), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.303464513Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-06T00:31:37.303435173Z }, migration_in: None, migration_out: None }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.337605297Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-06T00:31:37.303435173Z }, migration_in: None, migration_out: None }","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.339937956Z","hostname":"sock","pid":1572,"instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","propolis_id":"46330988-8b9f-455b-aa6c-07ae8d3bc524","instance_id":"c7df1963-d43c-46bf-b3d4-6a37670b026b","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.9210782Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63905"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.921185113Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e1e8b2e5-78b1-4ac3-81d1-0c61207662fe","remote_addr":"[fd00:1122:3344:101::c]:63905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.936161918Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {d04c5890-48aa-4e4d-951b-e3f160bd3d6f: {10.151.2.176: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}, 10.151.2.175: {d0428e7e-a355-4d59-a4c2-aa33fa6b3090}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.936184718Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e1290a87-db1a-4de2-8d8f-8f52ee112859","remote_addr":"[fd00:1122:3344:101::c]:63905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:37.944754233Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"42045a27-09ea-4d07-8f14-f9e6107b02dc","remote_addr":"[fd00:1122:3344:101::c]:63905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:38.048846753Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38932"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:38.049049739Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"11ae6142-deee-4ef9-8440-f5b4b8524da2","remote_addr":"[fd00:1122:3344:101::c]:38932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.530162213Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.531257706Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.531935066Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.532633307Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.533308007Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.535022907Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.535709058Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.536383388Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.538086158Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.54149711Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.543013075Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.543501319Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.544001864Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.544445577Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.545916411Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.546840318Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.547298362Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-06T00:31:38.548798587Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.501879243Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.501994516Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"647551ce-8bfa-4763-84b7-c4b6c8cbce8b","remote_addr":"[fd00:1122:3344:101::a]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.552910238Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.552996321Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.553020701Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"176a0b0b-732f-4629-a51c-549d3e5e0bbf","remote_addr":"[fd00:1122:3344:101::a]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.559102492Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.559491623Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"229aab25-c863-4271-a2a7-0720fb77af3d","remote_addr":"[fd00:1122:3344:101::a]:44280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":269,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:39.567633415Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ff2eacfb-71cd-428e-bf5f-5617462c866e","remote_addr":"[fd00:1122:3344:101::a]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":224,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:40.150231612Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63030"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:40.150334265Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0bd25342-613b-49c4-aa03-707e11a563e0","remote_addr":"[fd00:1122:3344:101::a]:63030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:40.164409473Z","hostname":"sock","pid":1572,"sled_id":"7c647f25-eb29-494d-840d-11658133af45","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:40.164431714Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"fdc84fbf-d4ec-41a5-baa0-2daa54d1aa2e","remote_addr":"[fd00:1122:3344:101::a]:63030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-06T00:31:40.172036009Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0bf5b586-82e4-42f0-b664-cee389b4b2e3","remote_addr":"[fd00:1122:3344:101::a]:63030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"204"}