[ Nov 21 01:13:19 Enabled. ] [ Nov 21 01:13:19 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Nov 21 01:13:19 Method "start" exited with status 0. ] [ Nov 21 01:13:19 Rereading configuration. ] [ Nov 21 01:13:19 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.668210874Z","hostname":"sock","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.668356916Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.721330753Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.721372423Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.835821316Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.845605304Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:19.853479985Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.455948675Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.593858978Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.593891608Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.59399927Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.594127762Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.594137802Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.594158732Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597399978Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:20.597411238Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597426589Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597444229Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597490749Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.59756673Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597692092Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597720293Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.597741713Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.609855404Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.609877064Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:20.710018435Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.197071629Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.197113199Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.1971227Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.19713566Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.1971495Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.19715548Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.19716924Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.20636939Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.20639386Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.298588989Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777064412Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777110483Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777141593Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777146703Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777155623Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777161903Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777191234Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777199884Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777212644Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777216564Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777220994Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777238275Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777242885Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777248965Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777264735Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777268085Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777273585Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777276585Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777279935Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777282945Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777296325Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777308136Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777313145Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777316656Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777321626Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777325256Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777343736Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777348576Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777356506Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777362336Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777404187Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777417177Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777424197Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777428957Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777456918Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777461408Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777464918Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777468538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777473008Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777487938Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777517368Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777521868Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777525959Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777530689Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777535179Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777539589Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777578769Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777583909Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777587189Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77759168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777595169Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77759949Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77762937Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77763383Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77764007Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77765114Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.77765458Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777688221Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777692641Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777695871Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777699391Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777742742Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777754482Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777762322Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777772292Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777779342Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777783582Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777812053Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.777826543Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.788828338Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:743"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.788872928Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999902049s","next_cleanup":"Instant { tv_sec: 1380, tv_nsec: 956704225 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.788925229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.78895847Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4005"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.78897682Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.78899388Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.792123774Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.793826518Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794461457Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1200"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794467277Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794609359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.79466963Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.79468485Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794691301Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794719351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794730251Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794751361Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794758471Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1305"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794761882Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794765322Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794793172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.794807252Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795104346Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-sled-plan.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795110086Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795265649Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-sled-plan.json\""} {"msg":"Sled plan written to storage: Plan {\n rack_id: e0f91ea1-f368-4705-9818-7568a8475d7c,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 623db7ea-cdeb-43ac-a5e8-f5c9bffff048 (sled),\n rack_id: e0f91ea1-f368-4705-9818-7568a8475d7c,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.7953842Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.79539061Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1339"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795430331Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795440921Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795603563Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/bootstore-network-config.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795609563Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795732235Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/bootstore-network-config.json\""} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795737135Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795795136Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 623db7ea-cdeb-43ac-a5e8-f5c9bffff048 (sled), rack_id: e0f91ea1-f368-4705-9818-7568a8475d7c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795901767Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4302"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.795963039Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4702"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.801086251Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.807221767Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.807348499Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811433437Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811492017Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811596979Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55544","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 623db7ea-cdeb-43ac-a5e8-f5c9bffff048 (sled), rack_id: e0f91ea1-f368-4705-9818-7568a8475d7c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811726961Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811734711Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811786651Z","hostname":"sock","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811791622Z","hostname":"sock","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811816642Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811821292Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:809"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811862543Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811901563Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811913003Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811923933Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:798"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811930653Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:826"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811934484Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:592"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811969464Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.811979344Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:697","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.828562208Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.84926345Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.866970149Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.894292224Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:407"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.894331735Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.894338195Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:419"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.894344395Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 108.699814ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.897461329Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81a3917d-d562-47f7-8388-5d1f329d6a0c\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"81a3917d-d562-47f7-8388-5d1f329d6a0c\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_0e00ae01-5cfa-4391-85ab-30571505fc6a/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.907685133Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_0e00ae01-5cfa-4391-85ab-30571505fc6a/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.931396797Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_0e00ae01-5cfa-4391-85ab-30571505fc6a/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.931431528Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.936712662Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:434"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.936722642Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_0e00ae01-5cfa-4391-85ab-30571505fc6a/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:21.936727982Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.007279626Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 133.409374ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.007798193Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eba05f7d-5908-49e6-9889-bbf0212bc9b2\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eba05f7d-5908-49e6-9889-bbf0212bc9b2\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.029271936Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.142463861Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 200.832262ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.143045539Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0d05338-7582-41b2-97ea-700034312a31\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0d05338-7582-41b2-97ea-700034312a31\" }"} {"msg":"Mounting oxi_0e00ae01-5cfa-4391-85ab-30571505fc6a/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.248009638Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.259867216Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.259936266Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.259965117Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.259979927Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.345070596Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 251.44294ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.345726855Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"556376be-9d6a-4e52-a1d9-a37d39f126d4\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"556376be-9d6a-4e52-a1d9-a37d39f126d4\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.597854029Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 106.804524ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.598574359Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f4f0de30-11bd-4452-83dd-ba6b9d139e64\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f4f0de30-11bd-4452-83dd-ba6b9d139e64\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.706281287Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 127.722661ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.706757563Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e20cf4c7-51be-43bb-a9c8-53b84c33ff07\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e20cf4c7-51be-43bb-a9c8-53b84c33ff07\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.834857859Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 437.183825ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:22.835447677Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b3c87504-a51d-45e5-9723-cb88994d4d48\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:13:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b3c87504-a51d-45e5-9723-cb88994d4d48\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.003115659Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.007956647Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:64014","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.007986638Z","hostname":"sock","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.008042369Z","hostname":"sock","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:64014","producer_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:23.009145554Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"270.553123ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.009965326Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.009984246Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.010037447Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:927","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.010042927Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:23.273313897Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:23.281787116Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"670.376439ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:23.953588133Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.058144561s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:25.013489188Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.122415233s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610159627Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2676"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610187167Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2921"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610192187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2546"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610196107Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2593"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610228988Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3002"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.610240908Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3047"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.612076364Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2846"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.613831608Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2587"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.61395608Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:25.614102442Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 28.971174ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:25.946684919Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 97.38504ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:26.159706261Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:26.160848437Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.21754006s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 230.697612ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:26.336854017Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 338.558882ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:26.685188086Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 759.457348ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:27.108078356Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.162540953s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:27.89959804Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.30318667s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:29.224362878Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:29.380208454Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.29154506s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.125038497s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:30.647350311Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:32.277066987Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.307015469Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.307490496Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308710263Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308734043Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308747053Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308768694Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308917196Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/sled-agent-request.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.308923676Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.309064698Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/sled-agent-request.json\""} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.309103478Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.309148319Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.309156649Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4005"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.30923682Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55544","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.512769868Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.51287105Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.512960681Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513015822Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513023052Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513035762Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513072563Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513086233Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.513116864Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.517745459Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4326","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.517763489Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4342"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.517773749Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.758962388Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37342"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.759391664Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.759397744Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.759483325Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.759510426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.759662978Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"496f6990-4686-4c8e-af2b-0ad77f9980ae","remote_addr":"[fd00:1122:3344:101::1]:37342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":378,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.789838863Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.789980575Z","hostname":"sock","pid":1566,"uri":"/sled-role","method":"GET","req_id":"52c1678b-4d35-4c16-939a-c33260e86e22","remote_addr":"[fd00:1122:3344:101::1]:63714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.791471346Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.791530717Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.791990434Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-service-plan-v5.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.791995054Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.792264227Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-service-plan-v5.json\""} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.792268947Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:980"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 67ce244a-1cf8-4843-81c8-9c6928631166, pool_id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 5dc4d6bc-c325-4794-8584-e388a1abded7, pool_id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: f42974ce-ea7b-4775-9d89-334a88f8d1da, pool_id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: c70f1cfa-7a77-450b-b325-2efad4444a92, pool_id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: bebd66df-8b61-47a8-b0be-bfe2bf789b52, pool_id: fad2b250-7ced-4cac-87db-22998050be7c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 2d97a7b8-9196-4ad0-b48c-562942b100bc, pool_id: f4bc5b75-b581-474b-87db-711856d8624e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 73935cd7-face-49ca-b675-1a76e55395bf, pool_id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 57ce847d-0f10-45fc-a805-5222dc269d45, pool_id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 25cb7f44-6871-4cb2-95f8-72d5c133afcb, pool_id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.823791331Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:382"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.823874313Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55946"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824134516Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:883"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 67ce244a-1cf8-4843-81c8-9c6928631166, pool_id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 5dc4d6bc-c325-4794-8584-e388a1abded7, pool_id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: f42974ce-ea7b-4775-9d89-334a88f8d1da, pool_id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: c70f1cfa-7a77-450b-b325-2efad4444a92, pool_id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: bebd66df-8b61-47a8-b0be-bfe2bf789b52, pool_id: fad2b250-7ced-4cac-87db-22998050be7c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 2d97a7b8-9196-4ad0-b48c-562942b100bc, pool_id: f4bc5b75-b581-474b-87db-711856d8624e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 73935cd7-face-49ca-b675-1a76e55395bf, pool_id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 57ce847d-0f10-45fc-a805-5222dc269d45, pool_id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 25cb7f44-6871-4cb2-95f8-72d5c133afcb, pool_id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824171847Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824187667Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824196687Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824210217Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1175"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824218737Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824222858Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824227548Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.824231258Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.98339532Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:32.983436541Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"db6f4d38-f0e6-466e-b782-b9022f769535 (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.189298152Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:33.190865034Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6210665456464903737"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:33.190975626Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12390111798954057366"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.286982669Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295750872Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295826623Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295844643Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295852163Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295857944Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295965305Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.295971875Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:33.308956328Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.332963726Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:33.340915558Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:34.311587457Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:35.31307921Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_db6f4d38-f0e6-466e-b782-b9022f769535","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.51738498Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.525818068Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.525883339Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.5258913Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.5258979Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.52590404Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.53587546Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.535905411Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.629488629Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638255343Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638284943Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638292053Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638307854Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638318294Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638446506Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.638460856Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.673280706Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.682803821Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.858597878Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.867135458Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.867172299Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.867179689Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.867186319Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.867204969Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.877590876Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.877611426Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.973611469Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.98155277Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981589961Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981597071Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981647612Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981667932Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981786724Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:35.981792044Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.01986627Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.029524846Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.208072922Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.216254818Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.216296749Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.216307809Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.216317329Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.216321959Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.226273639Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.22630471Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.324619725Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333030974Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333064954Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333071424Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333116045Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333124755Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333296367Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.333303007Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:36.337226043Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.369557188Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.378122759Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.552102361Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.560258756Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.560294516Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.560300446Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.560305986Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.560311196Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.570278087Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.570307887Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"fad2b250-7ced-4cac-87db-22998050be7c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.667257584Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674430625Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674465855Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674471965Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674521716Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674537316Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674678338Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.674684988Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.707390159Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.715307611Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_fad2b250-7ced-4cac-87db-22998050be7c","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.888103056Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.895916716Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.895965587Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.895973807Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.895983087Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.895989787Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.905124836Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.905158356Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f4bc5b75-b581-474b-87db-711856d8624e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:36.99913851Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006631576Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006664596Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006672466Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006681167Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006689207Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006821629Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.006828578Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.04240495Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.051057392Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_f4bc5b75-b581-474b-87db-711856d8624e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.226343292Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.233899918Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.233940789Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.233959819Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.233965899Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.233970779Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.243285541Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.243305881Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.338582574Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347199535Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347225266Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347230815Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347242006Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347249936Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347379928Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.347386728Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:37.351150651Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.382501983Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.390951062Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.569899064Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.57814879Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.57819241Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.57819991Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.57820791Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.578213661Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.587608153Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.587643254Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.681560687Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689303496Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689355577Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689363477Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689377527Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689390837Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.689546559Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.68955988Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.726578621Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.735172793Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_b39324e6-d2a2-455c-88e7-60bce779fce1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.913728259Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.922016946Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.922044396Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.922065236Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.922072236Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.922095147Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.932458823Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:37.932482513Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"6071668d-3100-464e-9aa6-4c653d46e04f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.033494846Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041438898Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041467919Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041474829Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041516739Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.0415317Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041658441Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.041663952Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.076995409Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.085241166Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_6071668d-3100-464e-9aa6-4c653d46e04f","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.263060172Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.270912182Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.270949083Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271025494Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271161576Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0e00ae01-5cfa-4391-85ab-30571505fc6a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67f3c571-2bea-4976-8ebe-c962b5aa082f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271226387Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271233337Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271297968Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-physical-disks.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271312048Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.2714997Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-physical-disks.json\""} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271561821Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:887"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271577441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271585612Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:913","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271590732Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:918"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271594902Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271609202Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271616532Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:929"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271643912Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271650082Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:934"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.271711503Z","hostname":"sock","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"9c96d476-0d98-4a05-a060-9e8ad477aaf4","remote_addr":"[fd00:1122:3344:101::1]:55946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5447747,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {037d4700-dc2a-4700-8a12-77f8fd3610c1 (dataset): DatasetConfig { id: 037d4700-dc2a-4700-8a12-77f8fd3610c1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 048fdaf6-f1dc-4b12-bb6f-db5aaa21653d (dataset): DatasetConfig { id: 048fdaf6-f1dc-4b12-bb6f-db5aaa21653d (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 04f19565-44fe-41ae-a9c7-389d8471ce3f (dataset): DatasetConfig { id: 04f19565-44fe-41ae-a9c7-389d8471ce3f (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 07156e4d-98ae-408d-8a86-0eb368f1b4cd (dataset): DatasetConfig { id: 07156e4d-98ae-408d-8a86-0eb368f1b4cd (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07aa58eb-cd16-4166-924b-ea92dd682e27 (dataset): DatasetConfig { id: 07aa58eb-cd16-4166-924b-ea92dd682e27 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ae171c2-bc55-4a81-9ac2-4f5a3c1b74c1 (dataset): DatasetConfig { id: 0ae171c2-bc55-4a81-9ac2-4f5a3c1b74c1 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0cbd2650-508d-44dd-875c-6c74d93c21bd (dataset): DatasetConfig { id: 0cbd2650-508d-44dd-875c-6c74d93c21bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0e42eec9-e7e9-43f7-9de5-a860bc863a08 (dataset): DatasetConfig { id: 0e42eec9-e7e9-43f7-9de5-a860bc863a08 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10f35c0d-824e-4c9b-836f-054b68d864f6 (dataset): DatasetConfig { id: 10f35c0d-824e-4c9b-836f-054b68d864f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 168ccadc-8b8f-4f69-8ab8-3e79a5710bb7 (dataset): DatasetConfig { id: 168ccadc-8b8f-4f69-8ab8-3e79a5710bb7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16c165eb-b937-4044-a99b-ce5060309f16 (dataset): DatasetConfig { id: 16c165eb-b937-4044-a99b-ce5060309f16 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b4f9d55-b15c-40e4-b551-5db3416ea8d1 (dataset): DatasetConfig { id: 1b4f9d55-b15c-40e4-b551-5db3416ea8d1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1b83eec3-b747-49c5-a3b5-38e6db0a4f8e (dataset): DatasetConfig { id: 1b83eec3-b747-49c5-a3b5-38e6db0a4f8e (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1d350b36-fb5e-41be-84fa-a83336a7754c (dataset): DatasetConfig { id: 1d350b36-fb5e-41be-84fa-a83336a7754c (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 267e6fb2-c400-4d7f-b76d-6d40e522f53a (dataset): DatasetConfig { id: 267e6fb2-c400-4d7f-b76d-6d40e522f53a (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2862de55-8349-4085-8c11-2178c0f24da4 (dataset): DatasetConfig { id: 2862de55-8349-4085-8c11-2178c0f24da4 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2dbd1c7f-93f6-466b-a874-79ef12dcd618 (dataset): DatasetConfig { id: 2dbd1c7f-93f6-466b-a874-79ef12dcd618 (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2fd71382-f5c1-4881-8162-cdd32e6b2669 (dataset): DatasetConfig { id: 2fd71382-f5c1-4881-8162-cdd32e6b2669 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 324398f9-1094-4df7-8e1d-de683ba20720 (dataset): DatasetConfig { id: 324398f9-1094-4df7-8e1d-de683ba20720 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 384d2ce1-9fc5-4ac9-ac2b-f0bf2a225043 (dataset): DatasetConfig { id: 384d2ce1-9fc5-4ac9-ac2b-f0bf2a225043 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3ec06cb5-6088-4b76-b84c-ff90158795e3 (dataset): DatasetConfig { id: 3ec06cb5-6088-4b76-b84c-ff90158795e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3fad4d0b-1983-4456-8561-9a708fb5221b (dataset): DatasetConfig { id: 3fad4d0b-1983-4456-8561-9a708fb5221b (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 438fea3f-8170-45bc-904c-b3d3725bea96 (dataset): DatasetConfig { id: 438fea3f-8170-45bc-904c-b3d3725bea96 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 456f54bb-1e9a-4e36-a52d-34c8c667787f (dataset): DatasetConfig { id: 456f54bb-1e9a-4e36-a52d-34c8c667787f (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b62f410-0a87-4db5-8c79-52cd0d0688b1 (dataset): DatasetConfig { id: 4b62f410-0a87-4db5-8c79-52cd0d0688b1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4e6bb5a7-26f9-419a-b797-2bd95a9709c7 (dataset): DatasetConfig { id: 4e6bb5a7-26f9-419a-b797-2bd95a9709c7 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5149f33e-8dff-4bf2-8117-653883eda43a (dataset): DatasetConfig { id: 5149f33e-8dff-4bf2-8117-653883eda43a (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cfcd6c4-c392-4717-b78f-b1d7a5d1f70a (dataset): DatasetConfig { id: 5cfcd6c4-c392-4717-b78f-b1d7a5d1f70a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6087fe5c-7fa3-429a-b182-219c8dca2e57 (dataset): DatasetConfig { id: 6087fe5c-7fa3-429a-b182-219c8dca2e57 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 65230a68-ada9-4774-93d0-c62ef9c5cf4a (dataset): DatasetConfig { id: 65230a68-ada9-4774-93d0-c62ef9c5cf4a (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69e8c60a-312a-4642-a38c-8a9a0d648365 (dataset): DatasetConfig { id: 69e8c60a-312a-4642-a38c-8a9a0d648365 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b5bed2b-e915-4515-9cbd-886646fca7c4 (dataset): DatasetConfig { id: 6b5bed2b-e915-4515-9cbd-886646fca7c4 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6dd71d4b-8cab-4634-9ab5-4abe27848416 (dataset): DatasetConfig { id: 6dd71d4b-8cab-4634-9ab5-4abe27848416 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e8bf656-8477-4c2f-8d1d-208e5c9bcbf6 (dataset): DatasetConfig { id: 6e8bf656-8477-4c2f-8d1d-208e5c9bcbf6 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 712d2be9-59b3-4ffd-8bfe-cdc40e4bbc09 (dataset): DatasetConfig { id: 712d2be9-59b3-4ffd-8bfe-cdc40e4bbc09 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 77078f97-0f29-4e3e-8c22-43c678d3d021 (dataset): DatasetConfig { id: 77078f97-0f29-4e3e-8c22-43c678d3d021 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7835380f-7592-474a-9339-79104bfea375 (dataset): DatasetConfig { id: 7835380f-7592-474a-9339-79104bfea375 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7e376fcd-5ded-40e9-beae-46da90e13c59 (dataset): DatasetConfig { id: 7e376fcd-5ded-40e9-beae-46da90e13c59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fce197a-86a7-4180-835e-0c6527cd530b (dataset): DatasetConfig { id: 7fce197a-86a7-4180-835e-0c6527cd530b (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 81034376-90fa-4376-8ae0-edef314e5c68 (dataset): DatasetConfig { id: 81034376-90fa-4376-8ae0-edef314e5c68 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8a0ab106-0f0b-4884-b772-0f727cf0bb26 (dataset): DatasetConfig { id: 8a0ab106-0f0b-4884-b772-0f727cf0bb26 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 910285b7-cab8-4a87-b748-686c8ffe3d55 (dataset): DatasetConfig { id: 910285b7-cab8-4a87-b748-686c8ffe3d55 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9b26cc80-9ac4-4196-b8d4-5cd832b6a4df (dataset): DatasetConfig { id: 9b26cc80-9ac4-4196-b8d4-5cd832b6a4df (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d2972ea-827b-4d87-8c31-da0d313aa41b (dataset): DatasetConfig { id: 9d2972ea-827b-4d87-8c31-da0d313aa41b (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a1184c3a-3e29-4a88-8fc7-26203ded50a8 (dataset): DatasetConfig { id: a1184c3a-3e29-4a88-8fc7-26203ded50a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b06e75b9-2ed2-43b2-91a5-e622763d894c (dataset): DatasetConfig { id: b06e75b9-2ed2-43b2-91a5-e622763d894c (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b09843e9-5c40-406a-ae22-27b85c5276d3 (dataset): DatasetConfig { id: b09843e9-5c40-406a-ae22-27b85c5276d3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b9b2db25-d32a-459d-83dd-32ef57231806 (dataset): DatasetConfig { id: b9b2db25-d32a-459d-83dd-32ef57231806 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd95d6dc-0241-40be-88c4-b5e2fcb80b09 (dataset): DatasetConfig { id: bd95d6dc-0241-40be-88c4-b5e2fcb80b09 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c4c54d19-281d-46c0-bd0f-33f760b4d992 (dataset): DatasetConfig { id: c4c54d19-281d-46c0-bd0f-33f760b4d992 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c524db76-d3d1-4e9f-a6d9-611f80dab669 (dataset): DatasetConfig { id: c524db76-d3d1-4e9f-a6d9-611f80dab669 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c689638d-9c2e-47b6-ba48-172bac3fbe5b (dataset): DatasetConfig { id: c689638d-9c2e-47b6-ba48-172bac3fbe5b (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cb551b58-474e-4f40-90b7-6938676c6c91 (dataset): DatasetConfig { id: cb551b58-474e-4f40-90b7-6938676c6c91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d0c21145-362c-41a4-b5b4-56689c3a2fc3 (dataset): DatasetConfig { id: d0c21145-362c-41a4-b5b4-56689c3a2fc3 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d1577707-51ae-4d34-8f00-02d87450389e (dataset): DatasetConfig { id: d1577707-51ae-4d34-8f00-02d87450389e (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d203d4d2-5550-49a5-9002-dae6116e1876 (dataset): DatasetConfig { id: d203d4d2-5550-49a5-9002-dae6116e1876 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, db9cb461-cd29-4fe5-b661-35b40f30a70d (dataset): DatasetConfig { id: db9cb461-cd29-4fe5-b661-35b40f30a70d (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ea0e092d-2cd9-4503-a217-9d2530744416 (dataset): DatasetConfig { id: ea0e092d-2cd9-4503-a217-9d2530744416 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ea54a39c-72f2-4146-bb9a-911fff415032 (dataset): DatasetConfig { id: ea54a39c-72f2-4146-bb9a-911fff415032 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eeb359e0-10f5-4066-9e1a-5e12966f33c3 (dataset): DatasetConfig { id: eeb359e0-10f5-4066-9e1a-5e12966f33c3 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, efc8470b-a481-42bd-aadb-c15568e34137 (dataset): DatasetConfig { id: efc8470b-a481-42bd-aadb-c15568e34137 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f05c0af7-11ba-4b04-a296-2281d3e86c1f (dataset): DatasetConfig { id: f05c0af7-11ba-4b04-a296-2281d3e86c1f (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f3d23492-bc9d-4544-a8b2-d924451b05ea (dataset): DatasetConfig { id: f3d23492-bc9d-4544-a8b2-d924451b05ea (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f6893e97-719b-43bc-94a2-0f26fec1b35b (dataset): DatasetConfig { id: f6893e97-719b-43bc-94a2-0f26fec1b35b (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f9893295-79ef-45be-af05-9cba93a3eb09 (dataset): DatasetConfig { id: f9893295-79ef-45be-af05-9cba93a3eb09 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffee1a11-61a5-42e9-8ead-9036d9400a27 (dataset): DatasetConfig { id: ffee1a11-61a5-42e9-8ead-9036d9400a27 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.303572152Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:457"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.303610553Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49281"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304185881Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:849"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {037d4700-dc2a-4700-8a12-77f8fd3610c1 (dataset): DatasetConfig { id: 037d4700-dc2a-4700-8a12-77f8fd3610c1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 048fdaf6-f1dc-4b12-bb6f-db5aaa21653d (dataset): DatasetConfig { id: 048fdaf6-f1dc-4b12-bb6f-db5aaa21653d (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 04f19565-44fe-41ae-a9c7-389d8471ce3f (dataset): DatasetConfig { id: 04f19565-44fe-41ae-a9c7-389d8471ce3f (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 07156e4d-98ae-408d-8a86-0eb368f1b4cd (dataset): DatasetConfig { id: 07156e4d-98ae-408d-8a86-0eb368f1b4cd (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07aa58eb-cd16-4166-924b-ea92dd682e27 (dataset): DatasetConfig { id: 07aa58eb-cd16-4166-924b-ea92dd682e27 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ae171c2-bc55-4a81-9ac2-4f5a3c1b74c1 (dataset): DatasetConfig { id: 0ae171c2-bc55-4a81-9ac2-4f5a3c1b74c1 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0cbd2650-508d-44dd-875c-6c74d93c21bd (dataset): DatasetConfig { id: 0cbd2650-508d-44dd-875c-6c74d93c21bd (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0e42eec9-e7e9-43f7-9de5-a860bc863a08 (dataset): DatasetConfig { id: 0e42eec9-e7e9-43f7-9de5-a860bc863a08 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10f35c0d-824e-4c9b-836f-054b68d864f6 (dataset): DatasetConfig { id: 10f35c0d-824e-4c9b-836f-054b68d864f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 168ccadc-8b8f-4f69-8ab8-3e79a5710bb7 (dataset): DatasetConfig { id: 168ccadc-8b8f-4f69-8ab8-3e79a5710bb7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16c165eb-b937-4044-a99b-ce5060309f16 (dataset): DatasetConfig { id: 16c165eb-b937-4044-a99b-ce5060309f16 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b4f9d55-b15c-40e4-b551-5db3416ea8d1 (dataset): DatasetConfig { id: 1b4f9d55-b15c-40e4-b551-5db3416ea8d1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1b83eec3-b747-49c5-a3b5-38e6db0a4f8e (dataset): DatasetConfig { id: 1b83eec3-b747-49c5-a3b5-38e6db0a4f8e (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1d350b36-fb5e-41be-84fa-a83336a7754c (dataset): DatasetConfig { id: 1d350b36-fb5e-41be-84fa-a83336a7754c (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 267e6fb2-c400-4d7f-b76d-6d40e522f53a (dataset): DatasetConfig { id: 267e6fb2-c400-4d7f-b76d-6d40e522f53a (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2862de55-8349-4085-8c11-2178c0f24da4 (dataset): DatasetConfig { id: 2862de55-8349-4085-8c11-2178c0f24da4 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2dbd1c7f-93f6-466b-a874-79ef12dcd618 (dataset): DatasetConfig { id: 2dbd1c7f-93f6-466b-a874-79ef12dcd618 (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2fd71382-f5c1-4881-8162-cdd32e6b2669 (dataset): DatasetConfig { id: 2fd71382-f5c1-4881-8162-cdd32e6b2669 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 324398f9-1094-4df7-8e1d-de683ba20720 (dataset): DatasetConfig { id: 324398f9-1094-4df7-8e1d-de683ba20720 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 384d2ce1-9fc5-4ac9-ac2b-f0bf2a225043 (dataset): DatasetConfig { id: 384d2ce1-9fc5-4ac9-ac2b-f0bf2a225043 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3ec06cb5-6088-4b76-b84c-ff90158795e3 (dataset): DatasetConfig { id: 3ec06cb5-6088-4b76-b84c-ff90158795e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3fad4d0b-1983-4456-8561-9a708fb5221b (dataset): DatasetConfig { id: 3fad4d0b-1983-4456-8561-9a708fb5221b (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 438fea3f-8170-45bc-904c-b3d3725bea96 (dataset): DatasetConfig { id: 438fea3f-8170-45bc-904c-b3d3725bea96 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 456f54bb-1e9a-4e36-a52d-34c8c667787f (dataset): DatasetConfig { id: 456f54bb-1e9a-4e36-a52d-34c8c667787f (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b62f410-0a87-4db5-8c79-52cd0d0688b1 (dataset): DatasetConfig { id: 4b62f410-0a87-4db5-8c79-52cd0d0688b1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4e6bb5a7-26f9-419a-b797-2bd95a9709c7 (dataset): DatasetConfig { id: 4e6bb5a7-26f9-419a-b797-2bd95a9709c7 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5149f33e-8dff-4bf2-8117-653883eda43a (dataset): DatasetConfig { id: 5149f33e-8dff-4bf2-8117-653883eda43a (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cfcd6c4-c392-4717-b78f-b1d7a5d1f70a (dataset): DatasetConfig { id: 5cfcd6c4-c392-4717-b78f-b1d7a5d1f70a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6087fe5c-7fa3-429a-b182-219c8dca2e57 (dataset): DatasetConfig { id: 6087fe5c-7fa3-429a-b182-219c8dca2e57 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 65230a68-ada9-4774-93d0-c62ef9c5cf4a (dataset): DatasetConfig { id: 65230a68-ada9-4774-93d0-c62ef9c5cf4a (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69e8c60a-312a-4642-a38c-8a9a0d648365 (dataset): DatasetConfig { id: 69e8c60a-312a-4642-a38c-8a9a0d648365 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b5bed2b-e915-4515-9cbd-886646fca7c4 (dataset): DatasetConfig { id: 6b5bed2b-e915-4515-9cbd-886646fca7c4 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6dd71d4b-8cab-4634-9ab5-4abe27848416 (dataset): DatasetConfig { id: 6dd71d4b-8cab-4634-9ab5-4abe27848416 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e8bf656-8477-4c2f-8d1d-208e5c9bcbf6 (dataset): DatasetConfig { id: 6e8bf656-8477-4c2f-8d1d-208e5c9bcbf6 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 712d2be9-59b3-4ffd-8bfe-cdc40e4bbc09 (dataset): DatasetConfig { id: 712d2be9-59b3-4ffd-8bfe-cdc40e4bbc09 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 77078f97-0f29-4e3e-8c22-43c678d3d021 (dataset): DatasetConfig { id: 77078f97-0f29-4e3e-8c22-43c678d3d021 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7835380f-7592-474a-9339-79104bfea375 (dataset): DatasetConfig { id: 7835380f-7592-474a-9339-79104bfea375 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7e376fcd-5ded-40e9-beae-46da90e13c59 (dataset): DatasetConfig { id: 7e376fcd-5ded-40e9-beae-46da90e13c59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7fce197a-86a7-4180-835e-0c6527cd530b (dataset): DatasetConfig { id: 7fce197a-86a7-4180-835e-0c6527cd530b (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 81034376-90fa-4376-8ae0-edef314e5c68 (dataset): DatasetConfig { id: 81034376-90fa-4376-8ae0-edef314e5c68 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8a0ab106-0f0b-4884-b772-0f727cf0bb26 (dataset): DatasetConfig { id: 8a0ab106-0f0b-4884-b772-0f727cf0bb26 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 910285b7-cab8-4a87-b748-686c8ffe3d55 (dataset): DatasetConfig { id: 910285b7-cab8-4a87-b748-686c8ffe3d55 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9b26cc80-9ac4-4196-b8d4-5cd832b6a4df (dataset): DatasetConfig { id: 9b26cc80-9ac4-4196-b8d4-5cd832b6a4df (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d2972ea-827b-4d87-8c31-da0d313aa41b (dataset): DatasetConfig { id: 9d2972ea-827b-4d87-8c31-da0d313aa41b (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a1184c3a-3e29-4a88-8fc7-26203ded50a8 (dataset): DatasetConfig { id: a1184c3a-3e29-4a88-8fc7-26203ded50a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b06e75b9-2ed2-43b2-91a5-e622763d894c (dataset): DatasetConfig { id: b06e75b9-2ed2-43b2-91a5-e622763d894c (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b09843e9-5c40-406a-ae22-27b85c5276d3 (dataset): DatasetConfig { id: b09843e9-5c40-406a-ae22-27b85c5276d3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b9b2db25-d32a-459d-83dd-32ef57231806 (dataset): DatasetConfig { id: b9b2db25-d32a-459d-83dd-32ef57231806 (dataset), name: DatasetName { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd95d6dc-0241-40be-88c4-b5e2fcb80b09 (dataset): DatasetConfig { id: bd95d6dc-0241-40be-88c4-b5e2fcb80b09 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c4c54d19-281d-46c0-bd0f-33f760b4d992 (dataset): DatasetConfig { id: c4c54d19-281d-46c0-bd0f-33f760b4d992 (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c524db76-d3d1-4e9f-a6d9-611f80dab669 (dataset): DatasetConfig { id: c524db76-d3d1-4e9f-a6d9-611f80dab669 (dataset), name: DatasetName { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c689638d-9c2e-47b6-ba48-172bac3fbe5b (dataset): DatasetConfig { id: c689638d-9c2e-47b6-ba48-172bac3fbe5b (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cb551b58-474e-4f40-90b7-6938676c6c91 (dataset): DatasetConfig { id: cb551b58-474e-4f40-90b7-6938676c6c91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d0c21145-362c-41a4-b5b4-56689c3a2fc3 (dataset): DatasetConfig { id: d0c21145-362c-41a4-b5b4-56689c3a2fc3 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d1577707-51ae-4d34-8f00-02d87450389e (dataset): DatasetConfig { id: d1577707-51ae-4d34-8f00-02d87450389e (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d203d4d2-5550-49a5-9002-dae6116e1876 (dataset): DatasetConfig { id: d203d4d2-5550-49a5-9002-dae6116e1876 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, db9cb461-cd29-4fe5-b661-35b40f30a70d (dataset): DatasetConfig { id: db9cb461-cd29-4fe5-b661-35b40f30a70d (dataset), name: DatasetName { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ea0e092d-2cd9-4503-a217-9d2530744416 (dataset): DatasetConfig { id: ea0e092d-2cd9-4503-a217-9d2530744416 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ea54a39c-72f2-4146-bb9a-911fff415032 (dataset): DatasetConfig { id: ea54a39c-72f2-4146-bb9a-911fff415032 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eeb359e0-10f5-4066-9e1a-5e12966f33c3 (dataset): DatasetConfig { id: eeb359e0-10f5-4066-9e1a-5e12966f33c3 (dataset), name: DatasetName { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, efc8470b-a481-42bd-aadb-c15568e34137 (dataset): DatasetConfig { id: efc8470b-a481-42bd-aadb-c15568e34137 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f05c0af7-11ba-4b04-a296-2281d3e86c1f (dataset): DatasetConfig { id: f05c0af7-11ba-4b04-a296-2281d3e86c1f (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f3d23492-bc9d-4544-a8b2-d924451b05ea (dataset): DatasetConfig { id: f3d23492-bc9d-4544-a8b2-d924451b05ea (dataset), name: DatasetName { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f6893e97-719b-43bc-94a2-0f26fec1b35b (dataset): DatasetConfig { id: f6893e97-719b-43bc-94a2-0f26fec1b35b (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f9893295-79ef-45be-af05-9cba93a3eb09 (dataset): DatasetConfig { id: f9893295-79ef-45be-af05-9cba93a3eb09 (dataset), name: DatasetName { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffee1a11-61a5-42e9-8ead-9036d9400a27 (dataset): DatasetConfig { id: ffee1a11-61a5-42e9-8ead-9036d9400a27 (dataset), name: DatasetName { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304354413Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304358854Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304362463Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304365634Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:906"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304368663Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.304372014Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:38.352913438Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.405112323Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.405144404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/debug\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.473486247Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.473518857Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/debug\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.541802309Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.54183878Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.60993258Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.6099632Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:38.674470229Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"19.221732546s"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.700867581Z","hostname":"sock","pid":1566,"name":"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.700902051Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone\"), full_name: \"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.768859319Z","hostname":"sock","pid":1566,"name":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.76889915Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\"), full_name: \"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.868353881Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.868389692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.958094026Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:38.958128007Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.061805888Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.061840988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.164654247Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.164685767Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.265184803Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.265231624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/debug\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.334373738Z","hostname":"sock","pid":1566,"name":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.334403259Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:39.355197152Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.424032732Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.424061873Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.531408565Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.531442446Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.630900507Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.630944648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.730745544Z","hostname":"sock","pid":1566,"name":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.730777615Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.820117144Z","hostname":"sock","pid":1566,"name":"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.820146454Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.908181135Z","hostname":"sock","pid":1566,"name":"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.908218415Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f4bc5b75-b581-474b-87db-711856d8624e/crypt/debug\"), full_name: \"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.976790481Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:39.976824822Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.08034113Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.080383351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.181342904Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.181372154Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.284241884Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.284271594Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:40.356792696Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.373602883Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.373628494Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.46634748Z","hostname":"sock","pid":1566,"name":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.466375901Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\"), full_name: \"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.567313603Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.567344253Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.679549725Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.679586465Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.78424211Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.78427151Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.874660504Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.874694395Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.977027537Z","hostname":"sock","pid":1566,"name":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:40.977059927Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\"), full_name: \"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone/oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.083242823Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.083274514Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.176335295Z","hostname":"sock","pid":1566,"name":"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.176364776Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\"), full_name: \"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.275897968Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.275929509Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone/oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:41.359249133Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.37744724Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.37747564Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.480347099Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.48037697Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.575974937Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.576016108Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.682890574Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.682933104Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/debug\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.751989887Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.752022738Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.841170184Z","hostname":"sock","pid":1566,"name":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.841203375Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone\"), full_name: \"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.910178087Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.910208077Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.981554513Z","hostname":"sock","pid":1566,"name":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:41.981585663Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone\"), full_name: \"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.019539268Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4350"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.019586168Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4360","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.026885951Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4374","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.026920492Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4439"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.06443923Z","hostname":"sock","pid":1566,"name":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.064470071Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.168322524Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.168359995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.185634798Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4505"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.185670879Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4532"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.230193856Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.230242997Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4391"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.271017741Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.271053382Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:42.278979714Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14837926199086426252"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:42.279130036Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6129436677629398958"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.284760005Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.284787786Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4512","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.30711076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4523"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.307174861Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:42.365629355Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.383168492Z","hostname":"sock","pid":1566,"name":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.383208473Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/debug\"), full_name: \"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.384732434Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4597"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.38518445Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.462264127Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.462299387Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.485506494Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4644"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.485549655Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.528548651Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:42.528919376Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.559414726Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.559449706Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.573683677Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.573797548Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:731"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.574449478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:734","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.574478528Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.622024258Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.65408614Z","hostname":"sock","pid":1566,"name":"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.65412236Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\"), full_name: \"oxp_f4bc5b75-b581-474b-87db-711856d8624e/crypt/zone/oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.758722694Z","hostname":"sock","pid":1566,"name":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.758765895Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6071668d-3100-464e-9aa6-4c653d46e04f/crypt/debug\"), full_name: \"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.828966944Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.828995535Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.900854157Z","hostname":"sock","pid":1566,"name":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:42.900892628Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), full_name: \"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/zone/oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.003414533Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.003449843Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.110201997Z","hostname":"sock","pid":1566,"name":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.110233248Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone\"), full_name: \"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.181119717Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.181150527Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone/oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:43.193492371Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6210665456464903737"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:43.193627503Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12390111798954057366"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.283194765Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.283227856Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:43.367058617Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.377021578Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.377032887Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.467813697Z","hostname":"sock","pid":1566,"name":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.467844297Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/debug\"), full_name: \"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.538395521Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.538422042Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.608904245Z","hostname":"sock","pid":1566,"name":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.608937916Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\"), full_name: \"oxp_6071668d-3100-464e-9aa6-4c653d46e04f/crypt/zone/oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.711660383Z","hostname":"sock","pid":1566,"name":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone/oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.711696134Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone/oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\"), full_name: \"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/zone/oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.81363683Z","hostname":"sock","pid":1566,"name":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.813673351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.905568046Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:43.905597626Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/zone/oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.009554921Z","hostname":"sock","pid":1566,"name":"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.009588852Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\"), full_name: \"oxp_fad2b250-7ced-4cac-87db-22998050be7c/crypt/zone/oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.112455421Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.112506142Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/debug\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.184931093Z","hostname":"sock","pid":1566,"name":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.184966233Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_db6f4d38-f0e6-466e-b782-b9022f769535/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.277239693Z","hostname":"sock","pid":1566,"name":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.277275224Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone\"), full_name: \"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347091258Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347428133Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347460673Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347729577Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json\""} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347809678Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347863429Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347870449Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347885049Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347891419Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347897529Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347904709Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347910549Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34791598Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.347922229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34792786Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34793336Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34793925Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34794336Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34795054Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34796507Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.3479708Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34797636Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34798069Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34798799Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34799207Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.34799597Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.348000861Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.348005421Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:851"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.348096402Z","hostname":"sock","pid":1566,"uri":"/datasets","method":"PUT","req_id":"64574292-936d-423e-8f98-e0a65db11178","remote_addr":"[fd00:1122:3344:101::1]:49281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6044351,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:44.369543034Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380463008Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.38060672Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36690"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380878724Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380927115Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380934365Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380984595Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.380993555Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.381212539Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.381352631Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.381419101Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.432495761Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.432556262Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.432563372Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.486052316Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.490331416Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.494221951Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.505668883Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.510144575Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:44.514543408Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:44.530293659Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/623db7ea-cdeb-43ac-a5e8-f5c9bffff048)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:45.371467093Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:46.078835472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:46.078919993Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:46.079098315Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:46.570742124Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:47.056704462Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:47.056847454Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:47.056870264Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 50.521925ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.064823396Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:47.064877107Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:47.068792132Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 123.51252ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.128366582Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 52.738747ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.198315847Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 47.606985ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.205149044Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 92.106285ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.259595031Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 229.902699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.271045892Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 129.868633ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.339049551Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 254.277899ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.398780882Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 159.554167ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.514129038Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 288.594364ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.530161414Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 561.175392ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.680519803Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 344.414535ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.691937804Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 682.970964ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:47.829544493Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 647.032227ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:48.045011139Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 951.549883ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:48.253465086Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 790.653601ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:48.522931363Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 964.440223ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:48.702737028Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 1.191634897s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:49.215564393Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 1.158984518s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:49.324025452Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 1.365695141s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:49.676028132Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 595.601848ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:50.416379105Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 753.966879ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:50.492165383Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 657.128776ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:51.022480746Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6\") failed: Property not found. retry in 1.161105228s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:51.051268922Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583\") failed: Property not found. retry in 1.119768438s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:51.255189936Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071\") failed: Property not found. retry in 1.466429027s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:51.688561303Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:52.281418277Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14837926199086426252"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:52.281553639Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6129436677629398958"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175365215Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_d8d6968b-3abb-4e6f-bece-80f196f71583"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175443586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_7476d0b8-adad-4918-b915-aae213b01071"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175454096Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_533221a7-87dc-40d6-9631-d7e9df3a16d6"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175461426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175809271Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.175834411Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.176095975Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.176212136Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"52cc3b20-7b47-4f14-b1a7-fdc399bdf50a","remote_addr":"[fd00:1122:3344:101::1]:36690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8795529,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.17856574Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:53.19492841Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6210665456464903737"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:53.195046102Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12390111798954057366"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.224109442Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.27794431Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.323627994Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:691"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.323638234Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.355062567Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.355799227Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.355804018Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.385509806Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.385637018Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35843"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.385929582Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.385955412Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.386071814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.386140155Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.386151085Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.386301757Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.386433449Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.38648175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.398440498Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.431448153Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.432225744Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.432234595Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.462228857Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 248, 201]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.462243187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.462697134Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.462702124Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.462957088Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.464252866Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.489498202Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:53.508755403Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:54.133412616Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:54.133548387Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 51.631596ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:54.262419323Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 111.576906ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:54.32319284Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 274.340477ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:54.446934504Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 517.218494ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:54.733724705Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 650.71765ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:55.261096107Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 976.242988ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:55.922129612Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 921.439382ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:56.907824002Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 729.420166ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:57.839006744Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25\") failed: Property not found. retry in 623.780399ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:58.577557051Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.221187141Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.221246272Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.221631828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.221664888Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.221929752Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.222043204Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"63ba7aaa-05ff-48fe-8ee4-11f9b1784efb","remote_addr":"[fd00:1122:3344:101::1]:35843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5836316,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.224439547Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:732"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.254633783Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.254809345Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:65250"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:13:59.266105964Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.266234186Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"38118f9c-3362-42b0-b5a0-d6a5cf132171","remote_addr":"[fd00:1122:3344:101::1]:65250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":11306,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:13:59.267890519Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.357868078Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:13:59.35803091Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44778"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:02.238534222Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12068735175743338107"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:02.282836556Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14837926199086426252"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:02.283144131Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6129436677629398958"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:02.397632944Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2202078118879610160"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.178854633Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16037417855825474973"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.196131336Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6210665456464903737"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.19637281Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12390111798954057366"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.366979274Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.367076905Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4f5acdef-d446-4f3a-919d-2e3b7399c6df","remote_addr":"[fd00:1122:3344:101::1]:44778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008989,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:03.368757619Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.490444194Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.490593326Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42591"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.499825936Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.499912357Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ac16f305-e762-42ff-98a7-265b1ca8cbf8","remote_addr":"[fd00:1122:3344:101::1]:42591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9264,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:03.502301751Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.72011201Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.720289223Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46391"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_68d7814c-2c5a-49b9-80f9-70dbb8694a25': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T01:14:03.728506749Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.728654591Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"67b1340f-3cab-48c1-8957-176745668cd0","remote_addr":"[fd00:1122:3344:101::1]:46391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8278,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:03.730210633Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.95352921Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:03.953686222Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:07.929370727Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"6956a0a7-ec67-471d-92a0-10b3f1a563c8","remote_addr":"[fd00:1122:3344:101::1]:49299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4008628,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1732151645.6424713, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:07.931815802Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:07.931823032Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.190815209Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.191000981Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.212819149Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8b3191d2-edab-4311-a673-f7ac47e50aca","remote_addr":"[fd00:1122:3344:101::1]:62995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21695,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2180654074, ip_addr: 129.250.35.250, stratum: 3, ref_time: 1732151647.9025362, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.215616218Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.215621138Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1425"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.245245226Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.245352707Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:45397"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.245794934Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.245872435Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.245996066Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.246076547Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.246092068Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.2462506Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.246391232Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.246453433Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280528843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280571573Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280582274Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280591934Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280599874Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.280718266Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f05b0760-358f-4cb6-a312-a5b183bf4f26","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":35153,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.280923958Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"138.403951ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f05b0760-358f-4cb6-a312-a5b183bf4f26\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f05b0760-358f-4cb6-a312-a5b183bf4f26\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420151171Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420512316Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420574397Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420729359Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420897561Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.420914071Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.421247646Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.421385458Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.421445049Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453052384Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453095255Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453104755Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453114035Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453123595Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.453196986Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"399fa585-4e0d-47d0-84cf-35e033e050bd","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized)]","latency_us":32881,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.453325198Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"86.831813ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"399fa585-4e0d-47d0-84cf-35e033e050bd\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"399fa585-4e0d-47d0-84cf-35e033e050bd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.541742014Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.542113169Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.5421748Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.542338202Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.542483124Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.542498795Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.542760418Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.543126713Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.543239905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.577606689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.57766609Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.577677811Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.577704241Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.577710041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.577774192Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bc1e4f73-7783-4ecc-a635-4f7a737cd7e4","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":35865,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.577914374Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"153.532427ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc1e4f73-7783-4ecc-a635-4f7a737cd7e4\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc1e4f73-7783-4ecc-a635-4f7a737cd7e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732107667Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732431431Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732497092Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732657675Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732816397Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.732834757Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.733132161Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.733315864Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.733365524Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766219907Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766263618Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766274718Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766284418Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766292278Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.76638716Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"dc274bfe-96a7-42c6-a2f7-d807b9e88d4b","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized)]","latency_us":34126,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.766531362Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"118.748497ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc274bfe-96a7-42c6-a2f7-d807b9e88d4b\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc274bfe-96a7-42c6-a2f7-d807b9e88d4b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.885987735Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.886268859Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.88631954Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.886493792Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.886604984Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.886652114Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.886875908Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.88701143Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.88706376Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921239292Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921267262Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921273042Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921278722Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921284593Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:08.921392174Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"acf832f8-3257-4244-a333-61435b20f7c7","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":35283,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:08.921557786Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"207.797659ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"acf832f8-3257-4244-a333-61435b20f7c7\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"acf832f8-3257-4244-a333-61435b20f7c7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.13038443Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.130677441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.130722713Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.1308764Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.131007182Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.131015161Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.131229917Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.131405493Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.131446949Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164492411Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164529828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164541124Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164548643Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164567582Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.164628773Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"207144ae-818a-4984-be49-358336906799","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized)]","latency_us":34030,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.164734552Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"283.464441ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"207144ae-818a-4984-be49-358336906799\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"207144ae-818a-4984-be49-358336906799\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.450698849Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.450920684Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.450947177Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451091901Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451220855Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451244256Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451382924Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451501547Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.451547429Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.485635152Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.485671837Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.48568019Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.485687015Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.485695002Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.485796845Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d8241183-8b3d-488b-9fb5-5816fd3bab80","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":34903,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.485956016Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"196.838746ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8241183-8b3d-488b-9fb5-5816fd3bab80\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8241183-8b3d-488b-9fb5-5816fd3bab80\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.683974487Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.684224624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.68426176Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.684410149Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.684528463Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.684550101Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.684890624Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.685117812Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.685164838Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718201201Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718237472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718246041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718252361Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718258444Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:09.718350771Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"3cea694b-0f67-41ff-bfe1-be3f5d3970ea","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":34202,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:09.718488061Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"520.128553ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3cea694b-0f67-41ff-bfe1-be3f5d3970ea\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3cea694b-0f67-41ff-bfe1-be3f5d3970ea\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241150836Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241369173Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.24139818Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241524661Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241652846Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241661294Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241816827Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241944413Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.241996426Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.276446163Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.276479286Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.27648732Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.276495101Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.276502526Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.276578568Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ad2ef39e-55ae-4cb7-8b81-3c8e2429d09c","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":35228,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.276687591Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"299.380845ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ad2ef39e-55ae-4cb7-8b81-3c8e2429d09c\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ad2ef39e-55ae-4cb7-8b81-3c8e2429d09c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.577913979Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578105118Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578148666Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578438655Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578594161Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578628108Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.578858127Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.579039236Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.579131881Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612150985Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612193661Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612219311Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612228189Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612235539Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:10.612368871Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6a11285c-3026-435a-91fd-b764374f0daa","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":34225,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:10.612479807Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"633.969971ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a11285c-3026-435a-91fd-b764374f0daa\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a11285c-3026-435a-91fd-b764374f0daa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.249601685Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.249847745Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.24987845Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.250082788Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.250180898Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.250192439Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.2503228Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.25042723Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.250471743Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285644051Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285685132Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285692201Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285697751Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285703265Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.285783909Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"96543d7c-6435-4c68-b459-dc45153cf275","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":35990,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.285902346Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"583.366957ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96543d7c-6435-4c68-b459-dc45153cf275\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"96543d7c-6435-4c68-b459-dc45153cf275\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872010315Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872259966Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.87231137Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872482973Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872677797Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872775513Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.872915371Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.873038148Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.873085109Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.906361743Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.906423983Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.906434952Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.906443831Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.906454556Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:11.906516835Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"62f94ff4-020f-4473-911e-bd2bca69e6bb","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":34334,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:11.90663784Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"403.197535ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"62f94ff4-020f-4473-911e-bd2bca69e6bb\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"62f94ff4-020f-4473-911e-bd2bca69e6bb\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.193494784Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.193740686Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"33afb150-2dce-44e0-acd6-5e0362ebbf59","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45317","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.311668595Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.311799022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.311824128Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.312117854Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.312283193Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.312330613Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.312549194Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.3127596Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.312861273Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.347726082Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.347779182Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.347789129Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.347798747Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.347806501Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:12.347896306Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"65627ee2-5b3f-4c8c-a47f-1722fed04875","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":36080,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.348084992Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"716.668058ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65627ee2-5b3f-4c8c-a47f-1722fed04875\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"65627ee2-5b3f-4c8c-a47f-1722fed04875\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:12.909401746Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"366.854561ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068043121Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068296737Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068335784Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068545018Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068683893Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.068717877Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.069043151Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.069290392Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.069351246Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105125739Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105205193Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105217625Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105227479Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105234847Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:13.105291614Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"3ef6016b-1ce0-4645-b775-cf3897d3ee2d","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized)]","latency_us":37036,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:13.105449847Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.345102777s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ef6016b-1ce0-4645-b775-cf3897d3ee2d\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ef6016b-1ce0-4645-b775-cf3897d3ee2d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.45519284Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.455487286Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.455519386Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.455824591Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.455989546Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.456005213Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.456227525Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.456404453Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.456487938Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.4915815Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.491626557Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.491635464Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.491642514Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.49166793Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:14.491729449Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"96d618f5-24f0-4d32-8d1e-83961e5ff7f4","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized)]","latency_us":36341,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:14.491912597Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.740189777s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96d618f5-24f0-4d32-8d1e-83961e5ff7f4\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"96d618f5-24f0-4d32-8d1e-83961e5ff7f4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.238265111Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.238614563Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.238666642Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.238940686Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.239119647Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.239164995Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.239438252Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.239698846Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.239760141Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275561598Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275672797Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275682341Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275691388Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275698259Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:16.275790604Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a6bda8c2-a34f-408c-a63c-3e0b3ed5b196","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized)]","latency_us":37247,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:16.275922156Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.000066867s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6bda8c2-a34f-408c-a63c-3e0b3ed5b196\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6bda8c2-a34f-408c-a63c-3e0b3ed5b196\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.282778157Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.28311537Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283136775Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283347574Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283475714Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283509774Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283664829Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283775886Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.283823306Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.317819099Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.317871525Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.317880919Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.317888119Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.317897109Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:18.317996561Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"2c2343d0-397c-43cf-bb84-9002d4039dee","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":35003,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:18.318213166Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.647842242s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c2343d0-397c-43cf-bb84-9002d4039dee\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c2343d0-397c-43cf-bb84-9002d4039dee\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.974578661Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.974783189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.974807181Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975059549Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975215964Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975247839Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975556479Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975681722Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:20.975741048Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.012679324Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.012725092Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.012734553Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.012740731Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.012746496Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:21.012808363Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"3ede27bd-957f-4e1a-8563-ef9be155b763","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":38010,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:21.01293787Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.964864145s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ede27bd-957f-4e1a-8563-ef9be155b763\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ede27bd-957f-4e1a-8563-ef9be155b763\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.983857599Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.98415406Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.984206026Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.984365001Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.984471905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.984496581Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.984725043Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.985034854Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:22.985132271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.018397095Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.01846499Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.018476605Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.018486309Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.018491691Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:23.01855801Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bc999311-dbc3-4e78-abc4-a1b4583a7501","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized)]","latency_us":34521,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:23.018680683Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.103732772s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc999311-dbc3-4e78-abc4-a1b4583a7501\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc999311-dbc3-4e78-abc4-a1b4583a7501\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.128906302Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.129219403Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.129251147Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.129420669Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.12964233Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.129652202Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.129876005Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.130119702Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.130217157Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.165999992Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.166042546Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.166050271Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.166057331Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.166065749Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:25.1661457Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"e6d6b730-aefe-4009-a9e1-816c86913caf","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized)]","latency_us":36985,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:25.166284602Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"3.772795831s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e6d6b730-aefe-4009-a9e1-816c86913caf\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e6d6b730-aefe-4009-a9e1-816c86913caf\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.35614238Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"277.19841ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.9502197Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.9504884Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.950546198Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.950711951Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.950811158Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.950828174Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.951114137Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.951301799Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.951386991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989397572Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989440511Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989448498Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989454236Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989458492Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:28.989533653Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"0ab4afea-05b8-4bab-9b06-48ff3fd0569a","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized)]","latency_us":39095,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:28.989677214Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.019316652s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ab4afea-05b8-4bab-9b06-48ff3fd0569a\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ab4afea-05b8-4bab-9b06-48ff3fd0569a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.021360831Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.021662663Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.021717217Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.02187366Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.022029999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.022044831Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.022248468Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.022440912Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.022505741Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.058842133Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.058904113Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.058911801Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.058920641Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.058929426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:33.059009348Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9acdd28b-ff29-4959-b6fd-51293b90906b","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":37455,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:33.059228827Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"3.466327321s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9acdd28b-ff29-4959-b6fd-51293b90906b\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9acdd28b-ff29-4959-b6fd-51293b90906b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535223066Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535430519Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535527897Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535653571Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535744359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535758891Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.535922656Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.536054273Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.53610503Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.572785827Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.572850741Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.572861409Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.572870588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.572879897Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:36.572961385Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"2768eaeb-1b75-4f57-bde5-d3111f6fb67a","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized)]","latency_us":37504,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:36.573113984Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"7.811801206s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2768eaeb-1b75-4f57-bde5-d3111f6fb67a\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2768eaeb-1b75-4f57-bde5-d3111f6fb67a\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:42.30979918Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:42.309920588Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"003b8ae0-12ca-4e08-aaf9-e593b74612a9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37098","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:43.709465558Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"531.613583ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.406617626Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.406836869Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.406855919Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.40703969Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.407137332Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.407164614Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.407297178Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.407415173Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.407472061Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.442981247Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.443033306Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.443051072Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.443058029Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.443062819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:44.44312147Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"35268610-bb7a-46b6-89b9-98a84eec827b","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized)]","latency_us":36291,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:44.443258103Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.679465443s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35268610-bb7a-46b6-89b9-98a84eec827b\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35268610-bb7a-46b6-89b9-98a84eec827b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.13645413Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.136824359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.136871291Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.137048819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.137156493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.137173631Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.137410052Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.137703159Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.13780382Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.172820138Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.172865308Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.172875142Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.172882754Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.172891726Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:49.17300345Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"3d1576e5-8773-49eb-82b8-f667cec7bcc9","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized)]","latency_us":36257,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:49.173185234Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.477717582s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d1576e5-8773-49eb-82b8-f667cec7bcc9\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3d1576e5-8773-49eb-82b8-f667cec7bcc9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664150964Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664414874Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664462959Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664631074Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664744701Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.664762101Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.665096675Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.665246424Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.665296075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.700846776Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.700891083Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.700899052Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.700969094Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.700977803Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:14:53.701055927Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"17edcccb-0d53-4d5e-8db6-1f7e7b72591a","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized)]","latency_us":36668,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:53.701200163Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"13.51034326s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17edcccb-0d53-4d5e-8db6-1f7e7b72591a\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:14:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"17edcccb-0d53-4d5e-8db6-1f7e7b72591a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:14:59.318906818Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.790253882s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249230757Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249523083Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249552108Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249840056Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249962542Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.249981442Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.250230052Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.25033197Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.250381573Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.288064252Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.288130365Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.288140518Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.288146003Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.28815294Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:07.288217169Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"66a9be32-26e9-4781-a44d-d75a555fefda","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized)]","latency_us":38740,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:07.288366721Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"8.631745332s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66a9be32-26e9-4781-a44d-d75a555fefda\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:15:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"66a9be32-26e9-4781-a44d-d75a555fefda\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:12.42586768Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57860"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:12.42595291Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7bda7c9d-d8ff-4fa5-b125-22ca92043e56","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57860","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.9440774Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.944424671Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.944476957Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.944695819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.944859884Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.944881906Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.945047706Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.945239998Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.945287118Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.980388332Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.980438489Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.98044765Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.980454839Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.980461637Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:15.980526026Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d234c850-7564-4172-b78c-2283302a6c35","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized)]","latency_us":36184,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:15.98066474Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"11.739016813s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d234c850-7564-4172-b78c-2283302a6c35\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:15:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d234c850-7564-4172-b78c-2283302a6c35\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:17.192900678Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.207921406s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752505004Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752655578Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752676184Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752824977Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752890415Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.752899668Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.75302478Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.753195934Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.753243635Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787441993Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787500054Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787519667Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787529417Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787534686Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3322","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:27.78761309Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"65e6446f-d433-4ce8-8937-65a5d682c85a","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\", TimeNotSynchronized), (\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\", TimeNotSynchronized), (\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\", TimeNotSynchronized), (\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\", TimeNotSynchronized), (\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\", TimeNotSynchronized)]","latency_us":34951,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:27.787767442Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"23.773891269s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65e6446f-d433-4ce8-8937-65a5d682c85a\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:15:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"65e6446f-d433-4ce8-8937-65a5d682c85a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:34.483402527Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"10.627979116s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:42.539877997Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:42.539956429Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"bc1ae6c8-0d58-4052-80b0-8250541738b1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61899","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.626990243Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.627180981Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.627198372Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.627444581Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.627570892Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.627588339Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.62787513Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.628047556Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.628106189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.716264856Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.716321154Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.716326291Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.716334532Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.716340897Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.804905206Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.809248911Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.813321919Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.817550497Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.822536244Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.827549669Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.831906251Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.836504279Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.841147787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:51.846187402Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.763734118Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.763943142Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.772534726Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.772806332Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.77282932Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.779117666Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.95368596Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.953948408Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 58.046264ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:55.9770103Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 59.082301ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:55.987831699Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.988083215Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 65.404357ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:55.996859284Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:15:55.996906019Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 137.789562ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.044135896Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 52.000849ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.2334686Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 97.157098ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.272073495Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 207.562477ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.286580612Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 244.797476ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.295969938Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 50.018597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.343564936Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 50.354726ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.375166492Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 129.198698ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.385977704Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 116.152605ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.403202567Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 102.200938ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.437228012Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 423.328201ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.509299875Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 418.687945ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.527734554Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 281.988853ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.538588205Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 252.50965ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.552866311Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 373.759732ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.565212725Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 336.90883ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.818537149Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 412.263505ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.837164625Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 412.470799ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.947340972Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 669.877173ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.957530299Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 910.34889ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:56.966552623Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 1.015773715s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.168403883Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 563.746283ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.270813496Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 981.115618ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.371652414Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 721.364011ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.639813936Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.479097054s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.845551423Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 1.266421044s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:57.991582658Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 657.4878ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:58.199764352Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 835.793967ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:58.365958466Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 815.512279ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:58.373781885Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 1.292958183s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:58.869422085Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 1.384011239s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.200722385Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 726.813318ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.212435151Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 1.035134903s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.2685742Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 647.39854ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.33884015Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 581.661805ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.950972621Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.137718495s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:15:59.996759923Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 601.21053ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.174746318Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.218132332Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"14.683708191s"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 947.273916ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.31520206Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 544.327987ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.542151641Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 1.143597262s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.597252122Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 696.377978ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:00.78748897Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 759.849787ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.096457334Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.038195563s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.146253764Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 816.761768ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.27459833Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 1.108448829s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.495179847Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 1.376072627s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.754589261Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 786.592363ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:01.869183664Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 693.308799ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.103841199Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 738.072789ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.199205974Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 1.344616974s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.615046783Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 858.363989ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.666711451Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 563.477459ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.805808931Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 583.414369ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:02.947325576Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 738.604744ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.140109453Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 864.871356ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.379032499Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 1.291097215s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.535014407Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.49987823s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.542553244Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 1.248201923s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.890859702Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 777.950513ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:03.968731199Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 652.613895ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:04.255304988Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 523.93548ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:04.757682127Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 1.327677822s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:04.836222034Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 611.580658ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:04.91756653Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 885.974993ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:05.050522844Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 1.402993909s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:05.147691083Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 831.259816ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:05.292357492Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 990.409968ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:05.539056308Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.13481302s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:05.947036738Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 871.36213ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:06.134020023Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 683.819834ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:06.173860494Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 551.909204ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:06.53904376Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d\") failed: Property not found. retry in 952.121634ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:06.560500983Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41\") failed: Property not found. retry in 904.895001ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:06.866712928Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f\") failed: Property not found. retry in 952.225042ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:07.013601368Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8\") failed: Property not found. retry in 1.321053287s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:07.091846051Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1\") failed: Property not found. retry in 1.407367265s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:07.101434936Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529619872Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_cockroachdb_cb2a7015-722e-46ca-8c50-302299ee299d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529686823Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_cockroachdb_bd001ad7-4d57-4a3c-901e-a38ac47924f1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529691783Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529695464Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_cockroachdb_e8128da2-fd7d-4f03-9c5a-f234b112ff41"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529699024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_cockroachdb_6755c857-e995-4f5c-94ed-a689ac8eaba8"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.529710134Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.530075469Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.530085519Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.530335053Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.530441744Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"4d665016-570b-48cf-afd7-202aed4f3e50","remote_addr":"[fd00:1122:3344:101::1]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":16875328,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.562994093Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34716"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.591453704Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"e85d6863-e3a6-424b-bf1e-b000001ee376","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28338,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:08.591676527Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"115.936011ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e85d6863-e3a6-424b-bf1e-b000001ee376\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e85d6863-e3a6-424b-bf1e-b000001ee376\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.735989471Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"d267071c-44f1-42ca-9443-c226d3df790f","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27042,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:08.736190094Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"130.626363ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d267071c-44f1-42ca-9443-c226d3df790f\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d267071c-44f1-42ca-9443-c226d3df790f\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:08.89843206Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"55a7d231-aa3d-4404-a758-f252ee60bac5","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":29936,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:08.898607912Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"192.105448ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"55a7d231-aa3d-4404-a758-f252ee60bac5\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"55a7d231-aa3d-4404-a758-f252ee60bac5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:09.119718368Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"91c3c0b4-f9e7-45d0-a2a9-96b06b204d98","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27849,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:09.119896161Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"174.269874ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91c3c0b4-f9e7-45d0-a2a9-96b06b204d98\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91c3c0b4-f9e7-45d0-a2a9-96b06b204d98\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:09.320785252Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"959f7d5c-99e4-48f6-b938-7504432020de","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25639,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:09.320967944Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"253.837761ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"959f7d5c-99e4-48f6-b938-7504432020de\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"959f7d5c-99e4-48f6-b938-7504432020de\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:09.601991524Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"633ba519-e922-414c-a2d9-00f1a3235141","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25475,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:09.602151117Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"160.952313ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"633ba519-e922-414c-a2d9-00f1a3235141\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"633ba519-e922-414c-a2d9-00f1a3235141\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:09.789560588Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"565dd4b7-8e0e-476e-af56-981ff4e76583","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25218,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:09.78971069Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"275.293083ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"565dd4b7-8e0e-476e-af56-981ff4e76583\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"565dd4b7-8e0e-476e-af56-981ff4e76583\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:10.091773107Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"25fbfd8f-c8d8-4339-80c1-27aec3ab682b","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25809,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:10.091927989Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"391.495552ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25fbfd8f-c8d8-4339-80c1-27aec3ab682b\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25fbfd8f-c8d8-4339-80c1-27aec3ab682b\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:10.51286083Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"05ac4ecc-99f8-46c4-b9bf-10736188dab5","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27561,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:10.513071613Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"379.852369ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05ac4ecc-99f8-46c4-b9bf-10736188dab5\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05ac4ecc-99f8-46c4-b9bf-10736188dab5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:10.922879598Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"dcb9a47f-2f03-4be9-a1a2-a6e496168e01","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_485f9f15-b5e1-460f-9ddf-26849b5b0e9f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28573,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:10.92305594Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"625.212442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dcb9a47f-2f03-4be9-a1a2-a6e496168e01\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 01:16:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dcb9a47f-2f03-4be9-a1a2-a6e496168e01\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:11.581481149Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3801"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:25.797710791Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.458340337Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3842"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.463576701Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"07136921-d71e-4e71-933c-90d10a331845","remote_addr":"[fd00:1122:3344:101::1]:34716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":17915895,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.466577394Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65270"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7476d0b8-adad-4918-b915-aae213b01071 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: d8d6968b-3abb-4e6f-bece-80f196f71583 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 533221a7-87dc-40d6-9631-d7e9df3a16d6 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9a6558f2-d733-4c53-aa26-d79aba7e08ab, kind: Service { id: 68d7814c-2c5a-49b9-80f9-70dbb8694a25 }, name: Name(\"ntp-68d7814c-2c5a-49b9-80f9-70dbb8694a25\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 201])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: cb2a7015-722e-46ca-8c50-302299ee299d (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: e8128da2-fd7d-4f03-9c5a-f234b112ff41 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: 6755c857-e995-4f5c-94ed-a689ac8eaba8 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd001ad7-4d57-4a3c-901e-a38ac47924f1 (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 485f9f15-b5e1-460f-9ddf-26849b5b0e9f (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 403cce04-ffba-4276-94a6-76cab9338e71 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 50ef7a5a-8210-43f1-a548-846251e25074 (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bbffc4db-58c5-4210-973d-49d52a55513e (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 49435a09-4a28-49d2-bf6d-81a21cb92fe2 (service), filesystem_pool: Some(ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e7f0b738-53b5-40fd-b5bc-91e44f045ca1 (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85ce7ec1-de81-4efe-9d6a-b287fd4011f9 (service), filesystem_pool: Some(ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b06b8807-db82-45ac-a20e-43cf3c78767c (service), filesystem_pool: Some(ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d2660ff4-a578-428c-a66f-05cf59317380 (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: d301b440-0189-4c47-8bc8-0d611caf344b (service), filesystem_pool: Some(ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External } } } }, OmicronZoneConfig { id: 852bb6ba-706d-41b0-9127-42d1c53fc0df (service), filesystem_pool: Some(ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External } } } }, OmicronZoneConfig { id: ea35cbf0-3332-47b2-8699-b75f196ec511 (service), filesystem_pool: Some(ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bba3a24-0a9b-4b1c-aa94-91f38078a3dd (service), filesystem_pool: Some(ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External } } } }, OmicronZoneConfig { id: f79dc41c-4e02-4884-87a2-0db5be12cf9b (service), filesystem_pool: Some(ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b2c9939-02f6-4f0f-845a-686a5ded8878 (service), filesystem_pool: Some(ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External } } } }, OmicronZoneConfig { id: c605ae15-9986-48f2-8313-f0f8aa776282 (service), filesystem_pool: Some(ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External } } } }, OmicronZoneConfig { id: 78a849c1-816d-4894-9879-1e069adabec4 (service), filesystem_pool: Some(ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7348eac2-3858-48ee-b01c-6a7a2dae7bdf (service), filesystem_pool: Some(ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.54228377Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.543823812Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35690"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.547072538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.547553445Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.548288215Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.549467502Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.549698785Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.551222676Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.551843755Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.552030138Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.674325571Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.733932551Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734149174Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734173525Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734181575Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734185545Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734191685Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734197245Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.734203345Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.747266869Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.926354003Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.926628767Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.967536743Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.967735496Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.967758086Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.967765906Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.996071695Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.996275298Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.996291958Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:29.996706464Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.002452915Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.008189436Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.014324113Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.02123623Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.026670396Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.032329536Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.037782973Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.171313255Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.175831218Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.180818379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.185134469Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.189668493Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.194648203Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.20006052Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.207491895Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.213198645Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.219611685Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.224973251Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.230823843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.238896347Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.246938231Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.253656765Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.26111449Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.269054112Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.277388749Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.287529652Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.296292176Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.388262782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.447183892Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.506467458Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.564011529Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.628906303Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.634146727Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.634167937Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.71099562Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 187, 138]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.71102944Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.713234962Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.713248282Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.778934937Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 234, 9]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.778965938Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.780388978Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.780400648Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.838292254Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 186, 142]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.838316794Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.840054379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.840071169Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.900818115Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 227, 174]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.900844975Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.902453208Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.902470288Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.958605079Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 137, 88]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.95863278Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.985031342Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.985048272Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.994368713Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.994383924Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.995752473Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.995766433Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.998122386Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:30.998137197Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.002363726Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.002376856Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.021005699Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.022459329Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.024823222Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.024839973Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.024850613Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.027377478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.029846763Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.029900874Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.029907734Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.146513107Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.279600743Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.291527931Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.306514252Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.315943395Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.328079576Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.343161128Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.355386071Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.369259836Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.380978851Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:31.392872519Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.50125543Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.508885588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.50900952Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.516009958Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.525291529Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.525566893Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.713913607Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.715947436Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.716151669Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.726744538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.728044116Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.728225719Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.728447042Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.736696418Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.745408871Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.74605941Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.746706009Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.747048034Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.747097875Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.758240362Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.766443167Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.772891848Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.780216611Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:38.787149369Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 53.662661ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:38.999094556Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 37.482841ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.050388289Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 51.842918ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.065164597Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 29.123222ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.132952732Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 68.159261ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.150158235Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 62.776684ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.2044437Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 70.713321ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.233753943Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 68.223392ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.2605298Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 89.519965ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.278651255Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 64.05815ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.296783531Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 143.952621ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.312633474Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 66.447148ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.327555504Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 221.403002ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.346797786Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 36.05948ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.362477187Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 39.352836ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.376411103Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 153.059068ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.390172647Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 54.865561ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.437336642Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 41.998535ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.448808683Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 127.06098ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.463624282Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 41.73464ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.4748157Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 142.780164ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.488814877Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 108.151937ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.554674865Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 109.753585ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:39.708710346Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 125.715493ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.012796451Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 76.746702ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.058621917Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 97.36742ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.106525902Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 520.701254ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.140217767Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 112.939394ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.164718602Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 112.612588ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.186639251Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 233.708336ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.210485027Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 81.427767ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.233318439Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 300.769202ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.247754492Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 292.416226ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.262170395Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 598.504692ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.274641701Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 392.240076ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.286600839Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 457.451263ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.299700814Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 294.340386ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.312285861Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 250.523724ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.323685362Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 151.54381ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.335795823Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:40.351547345Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 270.924495ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.360354549Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 267.86686ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.372474189Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:40.372876235Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 345.965255ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.461260051Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:40.489776113Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 391.379831ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.505588175Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:40.505800468Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 788.936841ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.566740297Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 263.21962ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.581205091Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 259.049397ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.60027433Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 445.463058ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.62512346Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 551.688922ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.650546638Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 210.410298ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.676792758Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 865.631479ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.702994747Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 450.065246ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.725460864Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 29.35614ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.742587165Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 536.607863ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.775865594Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 107.842604ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.79405349Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.089784834s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.824116154Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 72.416505ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.837836257Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 652.151483ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.870222454Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 874.548742ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.887099032Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.025087944s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.901604766Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.13234448s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.916550716Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 263.697338ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.927728934Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 944.838887ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.942144697Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 110.851923ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:40.955999912Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 168.680267ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.081301608Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.173295769s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.096158918Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 845.99546ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.219258412Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 591.330813ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.237524279Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 430.026231ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.252771874Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 355.134953ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.327222714Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.373224898s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.373461805Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.447729111s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.390883231Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 661.530518ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.540074303Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 845.080129ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.594902486Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.105003406s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.701594009Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.053994044s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.718008241Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 701.465713ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.779212333Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.019681923s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.84427114Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 567.150237ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.902690783Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 582.702412ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.928468716Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 662.784685ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:41.941696453Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 918.135858ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.063153264Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 926.867492ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.079785469Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.293054293s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.23103122Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.472502289s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.296182178Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.150642485s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.45456192Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 768.309176ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.484845137Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.182029255s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.563407874Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 728.750799ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.635420018Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.228844187s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.647484019Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.238208454s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.762618921Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 993.504286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.785376852Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 707.310449ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.821328318Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 824.190401ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.850802864Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 796.545779ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.875498812Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 608.583344ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:42.994102073Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 684.029348ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.020371453Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.172415938s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.267550986Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 707.043513ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.376696514Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.216025301s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.537963667Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.041292743s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.550400692Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 802.08746ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.616412983Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.23432282s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.627512759Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 880.560739ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.682834858Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 810.753531ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.693406807Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.00216666s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.716417692Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 667.168699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.758572016Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.452613895s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.781028192Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 830.908007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.791789154Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 659.193226ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:43.888807811Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 553.903169ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.013190684Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 901.443311ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.094792344Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 801.93578ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.704081Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 682.018844ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.732664112Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 723.090201ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.774783016Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.331094385s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.836846271Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.344839527s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:44.993776502Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.21370299s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.039366004Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 971.379226ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.080337312Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.445563961s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.096012443Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.351033489s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.148204078Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.418461697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.185195889Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 834.997441ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.229408843Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.231241648Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.385771068s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.288999652Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.146075213s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.301715971Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.226336829s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.43645248Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.436624532Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.436983517Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.381638579s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.523377945Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.402836406s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.536263916Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.53652246Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.586140119Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"26.253888419s"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 905.944286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.664590445Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.664812558Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.66492538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.665310785Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:45.671307679Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 61.821131ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.758484588Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 125.467991ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.832245207Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 39.508814ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.843149771Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 32.689345ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.870681829Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 59.441257ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.894311972Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 45.603536ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.911792678Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 127.877926ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.92115332Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 204.663703ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.964897237Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 241.377748ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.976616862Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 62.251725ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:45.98641233Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 268.090206ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.062279489Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 109.047999ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.073191913Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 991.116326ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.083885624Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 536.069649ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.091692743Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.379898982s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.176558899Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 578.40412ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.185979282Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 462.904756ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.194486372Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 534.949494ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.227444096Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 843.737115ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.262978667Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 272.58414ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.403503647Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 677.468447ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.413162553Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 603.87356ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.45835503Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.05502039s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.509124786Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 912.61189ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.553363189Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 954.43974ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.580554182Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 729.820117ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.612479312Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.19263707s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.63716774Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.086506327s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.669610147Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.2377088s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.678418501Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.102232551s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.686999302Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 815.806206ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.695050536Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 825.00832ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.772754051Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 569.146415ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.785423739Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 821.869867ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.915403911Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 969.303242ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:46.951790654Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 741.144687ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.072197831Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.208458437s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.085591779Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 638.877147ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.101166319Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.467580943s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.116211211Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 606.477364ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.351599448Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 971.681881ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.364757613Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 911.177363ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.47596292Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.244297001s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.520735661Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 700.376141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.544205032Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 858.581766ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.566009419Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 986.093874ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.57457545Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.164210436s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.606860575Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.066060548s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.746574234Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.365988451s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.754690408Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 963.405751ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.766384443Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 813.055737ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.797343629Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 823.827495ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.821490589Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 769.833503ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.838843619Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.094227188s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.925415364Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.045001573s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.93367353Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 954.921103ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:47.96701146Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 620.954019ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.25586074Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 771.658319ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.304352694Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 1.325087796s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.346229924Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 766.012986ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.396822017Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.026331378s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.434470867Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.117974657s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.570314891Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.263150305s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.594426811Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.435652821s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.617866901Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.235713674s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.625750043Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.241607207s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.654335806Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.424304648s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.738876927Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 939.714443ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.772787575Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.178332557s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.780726656Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 681.285853ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.822030959Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.382573426s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.887025185Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 916.748453ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.931541832Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.15609968s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:48.98819626Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.18323781s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.030630818Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.076427697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.086453085Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 741.314468ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.135610848Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.143203727s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.191564466Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 629.122995ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.471079235Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 811.993711ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.512747902Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 664.342523ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.681774164Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.408881204s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.697620387Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.11880815s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.721327881Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.199609834s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.857441859Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.452902774s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.86672384Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.288315997s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.875381172Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 603.152547ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.88517474Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 764.818338ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.905573288Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.095878796s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:49.967190046Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 587.174113ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.064603119Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 884.440953ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.109698804Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 823.858825ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.323570977Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.140737223s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.337159419Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 929.105067ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.354601855Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.029069548s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.36494188Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 714.901738ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.375709852Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 828.870862ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.389442736Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 735.834449ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.407851245Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.243530137s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.423351724Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 584.3192ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.500129545Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.45574797s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.662665726Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.203882569s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.679780437Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 893.409657ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:50.851083871Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 781.165962ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.005237174Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 578.108121ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.067046365Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 785.411969ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.074308747Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.076101354s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.096027943Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 773.435486ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.105590208Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 938.669816ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.115861102Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 932.004167ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.153781177Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.215479997s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.162073967Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.248036605s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.173312342Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.206099367s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.229602205Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 567.17199ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.294143834Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.327828189s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.331498011Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.223516261s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.404720413Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 661.90156ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.489410556Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:51.638935433Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:51.639090865Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 798.773529ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.65648421Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.310739256s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.680171214Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.267382599s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.756109994Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 630.972931ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.799448895Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.062223238s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.87646348Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 716.695834ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.886765045Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 58.363634ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.894657387Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.251924399s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.902911943Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 921.32154ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.911552315Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 113.469307ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:51.962445022Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 870.146484ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.074998188Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 294.636725ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.091954037Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 1.13867536s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.11484911Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.032370722s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.132800512Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 718.254595ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.176029232Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.285749874s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.260659064Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.438309205s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.388055409Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 321.87964ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.397739986Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 634.313173ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.430440397Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.070443225s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.439366502Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.277745719s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.446947079Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.221415992s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.463739176Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.21049467s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.611899414Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.485502863s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.637454404Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.029019898s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.674619878Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 515.291977ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.73157815Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 579.091041ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.844184127Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 808.432599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.904461426Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.009679635s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.949486411Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 552.554582ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:52.958730081Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.048312703s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.000475679Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.063817344s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.035142998Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.021324242s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.078070313Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 987.446932ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.167589224Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.211640911s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.177579925Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 559.743302ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.258292862Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 504.773704ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.269731573Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.392680768s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.437872503Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 785.370735ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.52424802Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.050855976s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.536102187Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.01993162s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.559579988Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 767.821873ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.698297003Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.155701133s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.716250936Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 886.601077ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.7271921Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 581.849267ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.740179833Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 1.134100507s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.789146436Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 515.074448ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.832747267Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.217824358s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.848424988Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.252972904s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.863221277Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.043086131s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:53.9755994Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.00299939s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.061653933Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 949.231035ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.109056241Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 549.087988ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.1189546Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 794.622332ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.134038723Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.097734924s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.167762678Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 735.795461ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.327496638Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 523.857286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.34462797Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 1.042158289s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.360679776Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.388077507s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.409034387Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.093538697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.49583903Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.399986103s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.591670041Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.003070083s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.603349576Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.467788659s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.628766203Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.183537405s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.68101081Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.241150606s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.844115998Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 899.51272ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.880778165Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 560.777665ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.892973877Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 1.290183757s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.933415337Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.350702925s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:54.943399617Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.362567999s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.034093155Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.095203364s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.075167694Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 955.544372ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.088041806Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 613.345061ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.101917471Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.496955602s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.118405363Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.447894573s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.1344492Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.386469269s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.281574542Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 1.478301872s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.414538366Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.480779797s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.467898818Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 581.956316ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.599965059Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 876.615845ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.619253521Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 760.581129ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.729471794Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 907.116836ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.791236575Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 703.504323ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.807061188Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 745.762502ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:55.879153143Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 571.305744ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.004951746Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 858.153704ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.058584162Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 658.895647ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.09816019Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.183689807s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.109110174Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.18069753s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.183584273Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.064927518s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.196353844Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 758.924417ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.23872219Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.128460651s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.307805754Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 611.714463ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.409301134Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.270595569s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.507176793Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 1.474472967s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.522017322Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.191606074s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.538500345Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.253638962s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.589354622Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.254429527s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.60342595Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.300373141s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.628784067Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 635.126236ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.643420383Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.293688665s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.681064974Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 765.572917ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.712768381Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.135743655s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.769363118Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 1.259010883s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.909592744Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.474424271s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.932183853Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.423746997s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:56.964181624Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 816.374236ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.017009708Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 543.47694ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.038905666Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 1.342108281s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.282830714Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 561.450367ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.30454803Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 523.54966ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.318976623Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 984.052224ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.380959797Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.018144816s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.455826801Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 859.614044ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.492440048Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.202347826s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.602506309Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 1.353808309s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.751793462Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 607.980826ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.793063874Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 708.56982ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.84812655Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 857.397132ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.862781706Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 580.648211ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.880582627Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:57.896617223Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43402"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.447569797s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.907200072Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.405923562s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.933923239Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:16:57.947813665Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e0a08a87-ffd6-451e-a022-42a1ba150f8e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43402","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25072,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b\") failed: Property not found. retry in 1.212338964s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.953797749Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.190902239s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.973405205Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 1.335006829s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:57.991189836Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 967.777439ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.01134263Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 940.352441ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.183201462Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.100858594s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.365341888Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 932.670801ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.378187559Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 849.973137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.400798058Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 943.480599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.417097158Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 697.403556ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.430220532Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.212062479s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.475069855Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 670.586054ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.48892915Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 1.384760889s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.56839902Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 911.503515ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.638312805Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.211760625s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.732938328Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 841.149536ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.817268607Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 546.462578ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:58.994099689Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 528.51791ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.11765676Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 904.816642ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.134113722Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 1.013712891s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.143342062Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef\") failed: Property not found. retry in 1.12500677s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.168643258Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4\") failed: Property not found. retry in 1.036132233s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.177828498Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd\") failed: Property not found. retry in 1.364846784s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.262909366Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c\") failed: Property not found. retry in 1.297578864s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.323504671Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 599.140475ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.336516554Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf\") failed: Property not found. retry in 1.129857759s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.351314942Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be\") failed: Property not found. retry in 1.278778887s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.36815883Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2\") failed: Property not found. retry in 1.191703271s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.37741579Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878\") failed: Property not found. retry in 1.222653059s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.477135965Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9\") failed: Property not found. retry in 539.001ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.551856418Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074\") failed: Property not found. retry in 818.764087ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.561940791Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 534.33681ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.656606234Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71\") failed: Property not found. retry in 1.490413581s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.669468096Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380\") failed: Property not found. retry in 1.110323265s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.699669381Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b\") failed: Property not found. retry in 809.76366ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.946257476Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511\") failed: Property not found. retry in 1.234326652s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.956115935Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e\") failed: Property not found. retry in 720.217636ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:16:59.966739045Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 803.805519ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:17:00.04867526Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df\") failed: Property not found. retry in 752.039328ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:17:00.169259299Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282\") failed: Property not found. retry in 643.269748ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:17:00.208195231Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1\") failed: Property not found. retry in 561.349083ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:17:00.880344099Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.48852729Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_external_dns_403cce04-ffba-4276-94a6-76cab9338e71"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488782084Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_pantry_b06b8807-db82-45ac-a20e-43cf3c78767c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488790774Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_852bb6ba-706d-41b0-9127-42d1c53fc0df"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488796914Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_oximeter_49435a09-4a28-49d2-bf6d-81a21cb92fe2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488805024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_ea35cbf0-3332-47b2-8699-b75f196ec511"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488810764Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_f79dc41c-4e02-4884-87a2-0db5be12cf9b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488816534Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_78a849c1-816d-4894-9879-1e069adabec4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488821814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_pantry_d2660ff4-a578-428c-a66f-05cf59317380"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488827454Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_nexus_5382fba6-edea-49d2-8ecc-9b1edfc688be"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488836494Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_0bba3a24-0a9b-4b1c-aa94-91f38078a3dd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488842034Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_7348eac2-3858-48ee-b01c-6a7a2dae7bdf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488847465Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_nexus_3285ddd9-4901-4d45-8459-dd50ba0635ef"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488852994Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_clickhouse_e7f0b738-53b5-40fd-b5bc-91e44f045ca1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488858405Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_d301b440-0189-4c47-8bc8-0d611caf344b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488864075Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_pantry_85ce7ec1-de81-4efe-9d6a-b287fd4011f9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488871565Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_external_dns_50ef7a5a-8210-43f1-a548-846251e25074"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488884885Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_4b2c9939-02f6-4f0f-845a-686a5ded8878"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488890115Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.488895745Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_nexus_bbffc4db-58c5-4210-973d-49d52a55513e"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.504221731Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.510826214Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.510859525Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.52119893Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.532590861Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"17bcf1e1-3595-4b8f-b639-fa5feb345260","remote_addr":"[fd00:1122:3344:101::1]:35690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31987383,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.573526218Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1448"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.576921835Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.577295371Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/rss-plan-completed.marker\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.577310611Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.577482543Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/rss-plan-completed.marker\""} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.857117474Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:787"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:01.888159881Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:799"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:02.023044012Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1013"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:03.835161098Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:03.907046982Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:03.985811941Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:04.172251369Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:04.318911595Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:04.630466516Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:04.988054285Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:05.224654679Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:05.748154906Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:06.194676578Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:06.688412276Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/e0f91ea1-f368-4705-9818-7568a8475d7c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:07.511449804Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88a0d826-a372-4d9e-92f9-842d9bfda447\", \"content-length\": \"173\", \"date\": \"Thu, 21 Nov 2024 01:17:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"e0f91ea1-f368-4705-9818-7568a8475d7c\\\"\", request_id: \"88a0d826-a372-4d9e-92f9-842d9bfda447\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:08.600642073Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f7b93a6-78f2-47b6-8dbb-9369100199d4\", \"content-length\": \"173\", \"date\": \"Thu, 21 Nov 2024 01:17:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"e0f91ea1-f368-4705-9818-7568a8475d7c\\\"\", request_id: \"7f7b93a6-78f2-47b6-8dbb-9369100199d4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:09.347899183Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"416d5b54-7d89-4145-b8f1-00d52f860fc6\", \"content-length\": \"173\", \"date\": \"Thu, 21 Nov 2024 01:17:10 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"e0f91ea1-f368-4705-9818-7568a8475d7c\\\"\", request_id: \"416d5b54-7d89-4145-b8f1-00d52f860fc6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:10.762069391Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d5da7cfa-2677-4ce2-ab44-742de3291836\", \"content-length\": \"173\", \"date\": \"Thu, 21 Nov 2024 01:17:11 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"623db7ea-cdeb-43ac-a5e8-f5c9bffff048\\\"\", request_id: \"d5da7cfa-2677-4ce2-ab44-742de3291836\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:12.634372946Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.233403587Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.324156286Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64700"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.332170099Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.332424932Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.333043611Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.335609648Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.347803829Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.347891761Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.369515755Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.369574736Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.390647423Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.390695774Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.41316259Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.413226691Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.433864312Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.433921023Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_fad2b250-7ced-4cac-87db-22998050be7c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.454348171Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.454413222Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f4bc5b75-b581-474b-87db-711856d8624e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.474010938Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.474062348Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.492861333Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.492926414Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.512622942Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.512672303Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:13.524107544Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"49e07710-57d3-41ce-8fe9-b1532339c272","remote_addr":"[fd00:1122:3344:101::c]:64700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195450,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.249141433Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54807"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.257064964Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.257226827Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.25817805Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.25819618Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.25820064Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.258235601Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.258242151Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.258271561Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c08ca724-3cfd-49b4-bc9f-65e3b9ec0834","remote_addr":"[fd00:1122:3344:101::a]:54807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8995,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.258973121Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1085"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.544385383Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53719"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.544533945Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.544547566Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.544591076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.54487776Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.556281991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.556341042Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.578628556Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.578682237Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.597605893Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.597653964Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.616269276Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.616314367Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.633198995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.633255976Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_fad2b250-7ced-4cac-87db-22998050be7c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.650185354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.650242915Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f4bc5b75-b581-474b-87db-711856d8624e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.666555195Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.666606156Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.685709175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.685758055Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.701889263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.701939924Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.71179584Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"bc278299-7eb2-4844-ba2e-d5e01af9af70","remote_addr":"[fd00:1122:3344:101::a]:53719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167334,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.920800638Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33243"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921099132Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921156963Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921193353Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921229264Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921246614Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921252144Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921269994Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:15.921297345Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7c9e4781-d5ae-4cf9-aae6-413e03cb6491","remote_addr":"[fd00:1122:3344:101::a]:33243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":394,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.414638087Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39135"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.41484043Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415026302Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415061913Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415087843Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415098173Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415117613Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415136584Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.415160744Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fee2a1c1-94fd-4678-834b-81ce828b05c9","remote_addr":"[fd00:1122:3344:101::c]:39135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":445,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981524655Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62064"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981789919Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.98184894Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981881871Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981891641Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981905161Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981914441Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981938251Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:16.981957421Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"af826770-38b5-4893-b775-f1778fb1698e","remote_addr":"[fd00:1122:3344:101::b]:62064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":343,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:20.230599281Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:20.231328571Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b6c6c846-0e51-44a4-a30b-c8b5ac0d3935","remote_addr":"[fd00:1122:3344:101::a]:45536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":658,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:20.257033614Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:20.257157746Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e990ee6f-b932-46a4-98f9-4b35e819b3ce","remote_addr":"[fd00:1122:3344:101::a]:45536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1182,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:20.346443784Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2378a14d-fc68-4127-9d6c-391eb938e6d8","remote_addr":"[fd00:1122:3344:101::a]:45536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4300,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:23.561308257Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:64014","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:53588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:23.654015884Z","hostname":"sock","pid":1566,"uri":"/623db7ea-cdeb-43ac-a5e8-f5c9bffff048","method":"GET","req_id":"ef9a5dc1-9aa7-41d6-a45b-4e398d4a2cb5","remote_addr":"[fd00:1122:3344:101::d]:53588","local_addr":"[fd00:1122:3344:101::1]:64014","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91638,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:27.983838089Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:27.983967051Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6a172970-fb95-44a0-b959-5e542a8bba98","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36683","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.573913197Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60303"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.574207141Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"69c90bd4-189a-400b-9546-819f0085edce","remote_addr":"[fd00:1122:3344:101::a]:60303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.647012457Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.647042158Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"81a5047b-1600-44b4-92c6-d25eeb336105","remote_addr":"[fd00:1122:3344:101::a]:60303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.65642018Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cad90741-7642-4d0d-ae2f-d17cd9736434","remote_addr":"[fd00:1122:3344:101::a]:60303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.762705368Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36327"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.763447378Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"d99dbe6b-0f49-419e-a2f5-fd66b242340e","remote_addr":"[fd00:1122:3344:101::a]:36327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":707,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.888961107Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41293"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.890124073Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:40.967657356Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.02824114Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9dc45d5f-65f6-4637-b4c9-0eddfb9437d3","remote_addr":"[fd00:1122:3344:101::a]:41293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139233,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.031485325Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7b3bf934-3552-4385-8fde-8c80240b13b0","remote_addr":"[fd00:1122:3344:101::a]:41293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2863,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.077732157Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.078315805Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.080924942Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/cluster/bootstore-network-config.json\""} {"msg":"Writing ledger to /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.080941952Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.082087738Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/cluster/bootstore-network-config.json\""} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.082119909Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.082146499Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"PUT","req_id":"8f6a31a0-204b-4bf7-bbed-a108142cc6c7","remote_addr":"[fd00:1122:3344:101::a]:41293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8270,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.255503162Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.255590533Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3dec41bc-d125-4339-8703-c8b015ea2925","remote_addr":"[fd00:1122:3344:101::b]:46643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.388927262Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.388962473Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0c5527f7-e602-4c0c-9b93-d9635a999ff7","remote_addr":"[fd00:1122:3344:101::b]:46643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.399288438Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7c510602-eb9c-4965-871e-03fad248d0d0","remote_addr":"[fd00:1122:3344:101::b]:46643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.432595308Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56092"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.43277951Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b40a8784-435d-4d5e-8668-1f7018a43791","remote_addr":"[fd00:1122:3344:101::b]:56092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.501563379Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60318"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.50162279Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"316e7a72-ad74-4439-b80a-d7a13638a3f2","remote_addr":"[fd00:1122:3344:101::c]:60318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.540351656Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.541285749Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"4fd5cabe-15b1-4b54-aa87-657ce8d85f18","remote_addr":"[fd00:1122:3344:101::c]:49729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":257,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.550352997Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40273"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.550435298Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.585531853Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.647377584Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1998b925-9da6-46e9-a9b8-ab2951da7aa1","remote_addr":"[fd00:1122:3344:101::b]:40273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96960,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.647631218Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45623"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.647642848Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.647662368Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7407b77e-f2bc-446d-81a6-17077e546cb0","remote_addr":"[fd00:1122:3344:101::c]:60318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.647821981Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.653135295Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"96131767-653a-4228-993b-c453e9c56cfd","remote_addr":"[fd00:1122:3344:101::b]:40273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":668,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.658009974Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"bd26673b-220e-4fc4-9269-9d9139a687f8","remote_addr":"[fd00:1122:3344:101::c]:60318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.689549469Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.746340669Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"346e569c-d69b-4950-b289-7fcb4d5802f0","remote_addr":"[fd00:1122:3344:101::c]:45623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98573,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:41.746854746Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"94bc63b2-f916-4d71-9c27-6f890fef2266","remote_addr":"[fd00:1122:3344:101::c]:45623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:58.016036201Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:17:58.016175723Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"05c78631-a6e7-4afc-933e-8d50c23261d7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59486","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.67924819Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63123"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.679409392Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5d49068a-be6b-417e-986e-06411660c6c2","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.698332689Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.698350939Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"558a6c9f-b4d9-449c-9d76-9ed0042216d3","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.708065506Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ee4d8484-2a5e-4790-8374-e4b923785b2b","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.83893033Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.839134113Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"3a2439cc-d4b0-4bbf-a291-99f8f0ad9775","remote_addr":"[fd00:1122:3344:101::a]:61272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.974691874Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57717"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:10.974780755Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.011412951Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.085560366Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cdbf9c43-6fa8-4cb5-a66e-26ad4b2d7e2e","remote_addr":"[fd00:1122:3344:101::a]:57717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110802,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.085979952Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"70bf282e-6e36-4646-8615-6a87242dffcc","remote_addr":"[fd00:1122:3344:101::a]:57717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.207959391Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61601"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.208112443Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e6f85e09-f717-439f-bdca-be433a23d507","remote_addr":"[fd00:1122:3344:101::b]:61601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.254609228Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.254642898Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b9a02e6f-9609-4ffa-97b4-00a6666e2d98","remote_addr":"[fd00:1122:3344:101::b]:61601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.265747635Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"91ef6998-568b-445b-a846-d2e61b342fbb","remote_addr":"[fd00:1122:3344:101::b]:61601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.389879674Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.390083047Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"2beb8cb7-4774-4394-a722-47f0318ddf93","remote_addr":"[fd00:1122:3344:101::b]:45342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.427535955Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.427632666Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b97d328e-9f48-413b-b6d0-aaf99a333eb8","remote_addr":"[fd00:1122:3344:101::c]:43585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.495595904Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.495629854Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"af20e9a3-592b-4df5-a353-e80ffb7d2bd0","remote_addr":"[fd00:1122:3344:101::c]:43585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.503942041Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1c520841-905e-4fab-8c85-93f9b7640163","remote_addr":"[fd00:1122:3344:101::c]:43585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.546163076Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60666"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.546218587Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.585242457Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.644001615Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6ea5636a-5e17-404e-bf3e-e5b848d77f0a","remote_addr":"[fd00:1122:3344:101::b]:60666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97802,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.644225988Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e8205044-31ee-4e9b-826b-5ede736eedf7","remote_addr":"[fd00:1122:3344:101::b]:60666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.676710946Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.676923229Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"84f5e55c-0b47-4cdb-abdd-465fa8c994b1","remote_addr":"[fd00:1122:3344:101::c]:56400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.747159079Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46486"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.747323891Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.782181372Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.840666166Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e01df4e8-d951-4992-9ec8-8e157c56eb62","remote_addr":"[fd00:1122:3344:101::c]:46486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93400,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:11.841185914Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7461e1a8-cd9c-454d-a6fb-6c3441fdb2a7","remote_addr":"[fd00:1122:3344:101::c]:46486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":290,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:28.051215445Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42418"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:28.051373397Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0bf0a4f6-6a51-448e-8ff7-19b3f833dec7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42418","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:38.542073831Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.58569326Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.585832772Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"31651d94-2729-43ae-890a-8b4a73d42ae4","remote_addr":"[fd00:1122:3344:101::a]:33421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.633869359Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.633899439Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c494d0b3-63a8-469b-bdc7-776b2cb992ee","remote_addr":"[fd00:1122:3344:101::a]:33421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":245,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.641941912Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0174f46c-9d48-4fc0-8b69-a2808bdf36f2","remote_addr":"[fd00:1122:3344:101::a]:33421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.754138554Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52518"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.754308446Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f91035e3-5fc3-4fdf-8c70-ccbaf24de6ab","remote_addr":"[fd00:1122:3344:101::a]:52518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.844520037Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49952"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.844592388Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.879621112Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.932611159Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3f398042-5a14-44e7-a1f5-032925e2c720","remote_addr":"[fd00:1122:3344:101::a]:49952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88043,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:40.932894963Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5d64fa09-628e-4f46-8db4-4c1809b37407","remote_addr":"[fd00:1122:3344:101::a]:49952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.16187601Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.161964741Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"beec8958-e31b-4c49-a1dc-544957c4ebd0","remote_addr":"[fd00:1122:3344:101::b]:36767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.22012771Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.220165521Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7be97da2-1b5a-4845-8d5f-923303cb4fc8","remote_addr":"[fd00:1122:3344:101::b]:36767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.229271969Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"856a340b-8919-44c8-a089-fb8db6e760e2","remote_addr":"[fd00:1122:3344:101::b]:36767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.376532574Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.376747587Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"4dd83155-326e-4d0f-ba49-5e7e7deef392","remote_addr":"[fd00:1122:3344:101::b]:45129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.437351021Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41349"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.437478703Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.478216757Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.541111743Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f475b36c-7786-4ed2-918c-2d5cd516b31e","remote_addr":"[fd00:1122:3344:101::b]:41349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103698,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.541359557Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.541646071Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"580a8244-cf04-4777-bc64-0e02af2e075d","remote_addr":"[fd00:1122:3344:101::c]:58801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.541687102Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7ebf9586-3f16-4ec3-8a0e-8be475c39061","remote_addr":"[fd00:1122:3344:101::b]:41349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":274,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.556119215Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.556139345Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f834f713-e5f0-4e17-a581-cd7f61827c71","remote_addr":"[fd00:1122:3344:101::c]:58801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.566040965Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ea4ecabb-825a-4e1b-927e-d823d775849a","remote_addr":"[fd00:1122:3344:101::c]:58801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.568254706Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58900"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.568384878Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.605323068Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.665652968Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"41b22c86-1aef-46f5-9065-0a493bb26b0d","remote_addr":"[fd00:1122:3344:101::c]:58900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.66579369Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.666079335Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"cb9f81b3-e068-4796-b80b-6bc824558705","remote_addr":"[fd00:1122:3344:101::c]:50098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:41.666196696Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e99a0397-617e-441d-a035-20ff875fe587","remote_addr":"[fd00:1122:3344:101::c]:58900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:58.084618684Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38290"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:18:58.084762586Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"57c0092e-028d-49dc-b104-b773c91e356b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38290","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:10.888084389Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:10.888199251Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f3a94449-ae77-4c8c-82f3-cc9a1f8f4aa3","remote_addr":"[fd00:1122:3344:101::a]:64676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:10.964940702Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:10.964973603Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"913c5562-af90-4ea2-99c6-8261ce5dd155","remote_addr":"[fd00:1122:3344:101::a]:64676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:10.97613105Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2ee2f7fe-1b53-4293-b62b-d7fdd3fe4db0","remote_addr":"[fd00:1122:3344:101::a]:64676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.268621332Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52967"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.268820924Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"d5428258-70b8-409a-9bba-cb5f9c1e2e95","remote_addr":"[fd00:1122:3344:101::a]:52967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.359163017Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.359287589Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c027b6c4-dfd9-4d6b-b12b-ce5051401d6f","remote_addr":"[fd00:1122:3344:101::b]:33877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.396441283Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.396518274Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"edc08160-3b79-4a1e-a805-ee640d0ea5da","remote_addr":"[fd00:1122:3344:101::b]:33877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.416741059Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ffcf6e44-5a5b-4702-83ce-e024ba38bd1d","remote_addr":"[fd00:1122:3344:101::b]:33877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":223,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.534123873Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34454"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.534304966Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"e333cd62-fc1e-41da-bf64-833b991ea1e9","remote_addr":"[fd00:1122:3344:101::b]:34454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.562246139Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58883"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.562331211Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.603603832Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.66729699Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"01e16a3b-ac1d-4ead-986f-a4f869450402","remote_addr":"[fd00:1122:3344:101::a]:58883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104989,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.667434802Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55481"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.667494333Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.667721676Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ddb71920-b555-4794-8746-74cf67629528","remote_addr":"[fd00:1122:3344:101::c]:52316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.667951429Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ae07fa24-a0e2-48ab-b4a4-b96f9f06819d","remote_addr":"[fd00:1122:3344:101::a]:58883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":278,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.667967749Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.708577841Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.712381725Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.712463236Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"27314a59-6853-447a-af34-15d5fc472f89","remote_addr":"[fd00:1122:3344:101::c]:52316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":663,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.721295521Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fc46224c-c1d4-4072-b737-8791d5843af5","remote_addr":"[fd00:1122:3344:101::c]:52316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":287,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.764303007Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"314e7ced-ad6d-49be-87ba-c2c93dee80d3","remote_addr":"[fd00:1122:3344:101::b]:55481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96462,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.764710843Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1f6a5e69-e5ae-4e94-8805-fa679b1fce7d","remote_addr":"[fd00:1122:3344:101::b]:55481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.777881558Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.778066881Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"2f04e219-d855-4a29-bce9-6a934572b89d","remote_addr":"[fd00:1122:3344:101::c]:45631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.964433277Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64467"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:11.964540959Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:12.001470029Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:12.055943057Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"22bba7b2-26c9-42aa-94c3-d3684d672378","remote_addr":"[fd00:1122:3344:101::c]:64467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91450,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:12.056255341Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d084dc88-5170-4417-bae4-39b512050b66","remote_addr":"[fd00:1122:3344:101::c]:64467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:28.118246136Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:28.118386918Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"cf5b8647-96bc-40fc-ac03-134666f45f59","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44977","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.566456985Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.566578897Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b155c846-174c-4d34-b729-37ed1083b111","remote_addr":"[fd00:1122:3344:101::a]:62409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.637651978Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.637824551Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"90b92728-a2b4-40b8-b360-929a8a871548","remote_addr":"[fd00:1122:3344:101::a]:39368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.669747871Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.669770061Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"62b8bc9c-8537-47cf-8f5d-25ad42d0df91","remote_addr":"[fd00:1122:3344:101::a]:62409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.67821581Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"10d2e70b-3610-4c62-99ef-5ed6fe300a19","remote_addr":"[fd00:1122:3344:101::a]:62409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.771775188Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38401"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.771852279Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.809850555Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.867072361Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"132a16ea-37de-48aa-b1d8-46046d7882c1","remote_addr":"[fd00:1122:3344:101::a]:38401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95248,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:40.867329365Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"21faeaba-8f7f-4632-b21d-500ccc28cbed","remote_addr":"[fd00:1122:3344:101::a]:38401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.315247996Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.315339838Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"90a52870-c935-4d29-b181-277e6613c405","remote_addr":"[fd00:1122:3344:101::b]:65152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.46098724Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.461224714Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c0089d16-bf02-418f-a9f1-bdc5004cba88","remote_addr":"[fd00:1122:3344:101::b]:48446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.487353342Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43856"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.487427453Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7993ee68-2052-4b3b-8af7-6636b0a0f81a","remote_addr":"[fd00:1122:3344:101::c]:43856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.496741494Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.496760425Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"67983ec8-cf80-40ad-a4d2-4d009e242e3f","remote_addr":"[fd00:1122:3344:101::b]:65152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.498313656Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44344"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.498403368Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.540596012Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.59862388Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"06e82609-646a-4dce-ae94-d37c9dc4f3a6","remote_addr":"[fd00:1122:3344:101::b]:44344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100242,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.599198518Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.599233058Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9d17463b-7ec1-4f51-a0dd-9e49deaa9689","remote_addr":"[fd00:1122:3344:101::c]:43856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.599297559Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"9c1665dc-110e-480b-813c-a0eb88b251a9","remote_addr":"[fd00:1122:3344:101::b]:65152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":355,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.59933346Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cf83306a-da89-4733-9874-9c77f49adaef","remote_addr":"[fd00:1122:3344:101::b]:44344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":322,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.646500364Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.646660187Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"3c9f6b0d-37be-4ebc-924f-45fb2ff13cde","remote_addr":"[fd00:1122:3344:101::c]:37198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.652731342Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7b19a07d-8d66-4c97-9d15-b6e23cd75e90","remote_addr":"[fd00:1122:3344:101::c]:43856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.675354881Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49903"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.675446692Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.71356106Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.770221768Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0293d8a0-2d3f-4aae-9eaa-23f70dc9e439","remote_addr":"[fd00:1122:3344:101::c]:49903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94806,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:41.770468811Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a381fb04-bc9d-40c2-9951-a5b590384ec3","remote_addr":"[fd00:1122:3344:101::c]:49903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:58.15426462Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:19:58.154503264Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"53e4d297-e81c-4b0f-8b18-8246524637f6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58142","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.598509853Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.598636445Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"99bc9028-d2dd-48eb-ac07-ad8460aeabe9","remote_addr":"[fd00:1122:3344:101::a]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.759396331Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.759460041Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"785bedeb-eb36-44a3-9eda-f0f8cfd1b7ec","remote_addr":"[fd00:1122:3344:101::a]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.767342903Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6381724c-337f-49ca-85e4-dce002f65261","remote_addr":"[fd00:1122:3344:101::a]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.885082612Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:10.885251014Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"06127fb8-99c8-4f16-8f42-b18a68eb56f7","remote_addr":"[fd00:1122:3344:101::a]:43979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.216739196Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40886"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.216822927Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.25611092Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.354522777Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f4ecbafa-10eb-495b-9bb6-940606b015d4","remote_addr":"[fd00:1122:3344:101::a]:40886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137712,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.354848971Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.355156826Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"cfe078f2-256e-4366-91fd-35c8b80d2a87","remote_addr":"[fd00:1122:3344:101::b]:59252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.355236387Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b3b52508-f4c1-4f3c-9065-fab7bb984584","remote_addr":"[fd00:1122:3344:101::a]:40886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":308,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.548743504Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.548788705Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"cc7b877b-107d-4bfe-9516-b07b5ffaa40d","remote_addr":"[fd00:1122:3344:101::b]:59252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.551361811Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53935"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.551471582Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.592752544Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.652472806Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cf069b13-a03d-4e21-9383-ddc66a126ddb","remote_addr":"[fd00:1122:3344:101::b]:53935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101034,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.653052034Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"be3e9eff-8dfe-496a-987c-f0a0f0ef24af","remote_addr":"[fd00:1122:3344:101::b]:53935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":268,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.653109675Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"405e63a6-4bc7-4184-9dba-ddce607bd387","remote_addr":"[fd00:1122:3344:101::b]:59252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":394,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.66197637Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.662150802Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a47c9458-eb03-4ee0-bd43-f7c325e021c2","remote_addr":"[fd00:1122:3344:101::c]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.67972741Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.67975472Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"40e0ba49-8cd8-423d-9922-e79fbf958aba","remote_addr":"[fd00:1122:3344:101::c]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.689045871Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0b76b70b-362e-44f0-9ef6-b14aecaa793c","remote_addr":"[fd00:1122:3344:101::c]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.741303117Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43837"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.74149798Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"fd0c26c0-2268-4a01-9dd8-ff9f39a5f424","remote_addr":"[fd00:1122:3344:101::b]:43837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.781597175Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62787"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.781684967Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.819236696Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.878668873Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2b4146b2-a661-4bfe-9753-642f619b51d4","remote_addr":"[fd00:1122:3344:101::c]:62787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97011,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.878784005Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.878965528Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b178e03b-47a1-4053-8d7e-5f89f96bddcb","remote_addr":"[fd00:1122:3344:101::c]:41732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:11.87910871Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3def1b7f-4694-4bf5-8147-d40a0c5480d7","remote_addr":"[fd00:1122:3344:101::c]:62787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:28.187912332Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52680"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:28.188045824Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c70de619-c4eb-4ac0-92e1-744a8b4beaf5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52680","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.531895663Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55853"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.532019554Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a9b50542-9421-4102-a9b3-11d709abaa76","remote_addr":"[fd00:1122:3344:101::a]:55853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.631940073Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.632159076Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"eea24e5a-ad24-45f1-88e0-5c0cfbde46cd","remote_addr":"[fd00:1122:3344:101::a]:56178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.646614469Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53715"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.64670125Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.684347551Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.741181732Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"49cd54ac-ba52-429e-802f-d14212b136d5","remote_addr":"[fd00:1122:3344:101::a]:53715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94520,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.741460776Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.741470166Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"37464512-823f-4627-9a16-d89c4a0f274b","remote_addr":"[fd00:1122:3344:101::a]:55853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.741686459Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6a3c37b3-dd5e-45b7-8df3-04222dc2b89d","remote_addr":"[fd00:1122:3344:101::a]:53715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:40.752233788Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b51e0602-7fd2-45fe-8687-4d6ba01fe7c0","remote_addr":"[fd00:1122:3344:101::a]:55853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.262331066Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.263004766Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9f359765-ef82-419f-945f-bf5a8108a902","remote_addr":"[fd00:1122:3344:101::b]:36615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.311167194Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.311185025Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1b6bd88c-8414-41af-be78-03f9dd546679","remote_addr":"[fd00:1122:3344:101::b]:36615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.322386642Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c40bb4f3-9e30-407c-a27f-f55e7d5f1ceb","remote_addr":"[fd00:1122:3344:101::b]:36615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":215,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.386731419Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.386957802Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f3a42ff8-197d-40ed-bf76-890363f57ac9","remote_addr":"[fd00:1122:3344:101::b]:64835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.435472046Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60059"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.435556127Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.475918096Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.533581648Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7849af8d-9e96-4618-98b1-9c9cf10262cc","remote_addr":"[fd00:1122:3344:101::b]:60059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98058,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.533869143Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"16cc7423-9b96-4fdd-aa09-d63114b886b1","remote_addr":"[fd00:1122:3344:101::b]:60059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.540173252Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.540932492Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a871fbb9-a179-4a24-8a9e-b5ff78ce54e3","remote_addr":"[fd00:1122:3344:101::c]:49166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.590725194Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.590740524Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4192328e-fd0c-4627-8809-a342638138bb","remote_addr":"[fd00:1122:3344:101::c]:49166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.632091577Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.63229265Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"63facd99-0d8a-40c3-9a68-c7d2401a9974","remote_addr":"[fd00:1122:3344:101::c]:49827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.63518747Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b4a717a3-c7c5-47fc-a09f-db46be4b0f00","remote_addr":"[fd00:1122:3344:101::c]:49166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.662687638Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44323"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.662742679Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.699408605Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.755867401Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f500a06a-0270-4764-9fd6-fb6715ea9db3","remote_addr":"[fd00:1122:3344:101::c]:44323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93151,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:41.771053685Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"eff97c0f-2a2c-4260-9df6-52c9fdfb5ed7","remote_addr":"[fd00:1122:3344:101::c]:44323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:58.219977642Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:20:58.220114024Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a76f2bd6-6216-4f55-8cf6-46d2059d98d1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39174","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.74830812Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.748490953Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b2826f6a-3fd2-4d11-93d2-522652216ae9","remote_addr":"[fd00:1122:3344:101::a]:65493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.76814533Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.768212971Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b76d71f9-5422-4d66-b621-97a0320df5e6","remote_addr":"[fd00:1122:3344:101::a]:56868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.820038342Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.820056662Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"685ad468-85c1-4c70-a824-e1c2d60f5a0c","remote_addr":"[fd00:1122:3344:101::a]:56868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.830368117Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"52af5923-d93f-4196-a7b5-e573eef0341e","remote_addr":"[fd00:1122:3344:101::a]:56868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.860681934Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58693"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.860741865Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.901749663Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.955832945Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"002dfa1a-f5be-47fb-8c23-5991b61b6726","remote_addr":"[fd00:1122:3344:101::a]:58693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:10.956130869Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b35e9010-ef82-4b8d-81d2-4c1404bfcaba","remote_addr":"[fd00:1122:3344:101::a]:58693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.355384805Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.355505507Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f1dd2ced-ac00-43a2-87b0-b988cfb5e0cd","remote_addr":"[fd00:1122:3344:101::b]:61861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.368100874Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.368606741Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"622e29d4-5443-4a31-9553-53eb42755dfa","remote_addr":"[fd00:1122:3344:101::b]:52142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.434827715Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.434860675Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"51aef0a3-24f2-436b-a7d4-7b05de91fbe7","remote_addr":"[fd00:1122:3344:101::b]:61861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.444562422Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a0196307-edfd-4417-b130-bc5a257cd7ed","remote_addr":"[fd00:1122:3344:101::b]:61861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.577067429Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42029"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.57715364Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.615963047Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.672129719Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5eb6b9b3-0186-49b9-8c20-27828e992643","remote_addr":"[fd00:1122:3344:101::b]:42029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95013,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.674872268Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fd0d2726-75ac-4fbb-be24-e7ef50e98da6","remote_addr":"[fd00:1122:3344:101::b]:42029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.800752031Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.800888343Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a168de6d-0094-491d-9bf2-d9a9c7703c33","remote_addr":"[fd00:1122:3344:101::c]:57288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.844100862Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.844133513Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b28190a7-b073-4582-8ea2-cdc9cd51a189","remote_addr":"[fd00:1122:3344:101::c]:57288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.852375359Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1806cf58-638e-4ad8-95fb-f7ae3e759e5f","remote_addr":"[fd00:1122:3344:101::c]:57288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.865852649Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48155"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.86592901Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.902539626Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.955084096Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"86177100-d452-426d-a8df-b24ab34a1b3b","remote_addr":"[fd00:1122:3344:101::c]:48155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.955198208Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54996"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.95537441Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"1c3d5c8b-4c12-444f-9d57-a8f63cf6ce96","remote_addr":"[fd00:1122:3344:101::c]:54996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:11.955509982Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"bfaaa53a-9b21-47d5-ae3b-96aa229c82d1","remote_addr":"[fd00:1122:3344:101::c]:48155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:28.252317996Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:28.252425607Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"db85cb9b-0443-4658-ad5e-73de67f8e985","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46697","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.587973279Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.588110281Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9870a377-5b1e-4c7f-aa4e-e2efc61033ba","remote_addr":"[fd00:1122:3344:101::a]:64933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.63342161Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.63348652Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"39ee76c8-b55f-4807-b275-46df1aa401a2","remote_addr":"[fd00:1122:3344:101::a]:64933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.653796147Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61262"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.6540014Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"31fcc480-72e4-4ecd-ba97-24e69f325820","remote_addr":"[fd00:1122:3344:101::a]:61262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.656226721Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"67f651a5-dc07-44f0-9ac8-b8bfaa031440","remote_addr":"[fd00:1122:3344:101::a]:64933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.744354423Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37735"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.744446934Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.782801565Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.836817866Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"168c5f0c-54eb-4c35-943f-248ff4ba469f","remote_addr":"[fd00:1122:3344:101::a]:37735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92407,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:40.83714161Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1c56fe14-ac02-4a15-ae67-57c332dbc9bf","remote_addr":"[fd00:1122:3344:101::a]:37735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.268020242Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.268106503Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"44635790-3c02-48cf-b221-5476c807ce98","remote_addr":"[fd00:1122:3344:101::b]:60206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.282977573Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.282992403Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6e00a795-7be8-42a4-8811-c8f19c08b717","remote_addr":"[fd00:1122:3344:101::b]:60206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.295776973Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.295941846Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"58b232e4-5818-4ab9-a658-925b316f2673","remote_addr":"[fd00:1122:3344:101::b]:46503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.304540987Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"149c029f-1da2-44d6-9511-6887cd44f47f","remote_addr":"[fd00:1122:3344:101::b]:60206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.389693047Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40965"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.389784428Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.430176877Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.479779216Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5b437aef-235c-4c45-831c-eea386eaddc7","remote_addr":"[fd00:1122:3344:101::b]:40965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90025,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.48006368Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f3732071-3f4d-4e81-9d9d-b4e2fec94404","remote_addr":"[fd00:1122:3344:101::b]:40965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.529566668Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.52970322Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"99e9ddef-8b75-45cb-9f0e-c3ef137015db","remote_addr":"[fd00:1122:3344:101::c]:50726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.544671801Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.544687891Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0f77a0a6-d9a0-4c8c-b84d-f8240908347e","remote_addr":"[fd00:1122:3344:101::c]:50726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.554135354Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3ab22861-a090-4e93-b420-6d7317e8183f","remote_addr":"[fd00:1122:3344:101::c]:50726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.605009951Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.605180043Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"47ac3ec8-b0a8-428d-b31d-67b29aad1790","remote_addr":"[fd00:1122:3344:101::c]:51455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.634877342Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54338"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.635039304Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.671692231Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.726919119Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f1712b31-ad52-4abe-be4e-9fafc4f18ff5","remote_addr":"[fd00:1122:3344:101::c]:54338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91952,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:41.727408376Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b45f9c51-a76b-42e3-ae51-0bbba74fb3be","remote_addr":"[fd00:1122:3344:101::c]:54338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":244,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:58.285233647Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62728"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:21:58.285422449Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"da33df34-3b7a-431e-b5cb-20be41cb0824","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62728","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.757506714Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.757622686Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f2f07076-d442-490e-b378-6b6c2599b3dc","remote_addr":"[fd00:1122:3344:101::a]:56130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.807780023Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.807811003Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ce832f7d-058f-4f40-8381-b4352d9594da","remote_addr":"[fd00:1122:3344:101::a]:56130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.838668238Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.838852441Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c67fa7ad-e6d7-4902-9215-b484762f00cf","remote_addr":"[fd00:1122:3344:101::a]:59522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.848146912Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ccc94d17-074d-494e-a793-3743a3b4ce9a","remote_addr":"[fd00:1122:3344:101::a]:56130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.863043182Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46854"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.863124143Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.905243686Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.965549146Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8ce9ab42-0a4c-4b9b-98ec-df9cde78d79a","remote_addr":"[fd00:1122:3344:101::a]:46854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102452,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:10.96582509Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9a424f2e-f692-4dad-8788-4651df000272","remote_addr":"[fd00:1122:3344:101::a]:46854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.136624917Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48750"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.136907911Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.136961291Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.136985352Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.137006552Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.137031452Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.137083573Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.137096153Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.137119034Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7038caa9-484b-45d7-bb30-5eccc672bcf4","remote_addr":"[fd00:1122:3344:101::a]:48750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":385,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.364068212Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.364265254Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"15cdc008-c163-4edb-b0a0-b225f83ebf50","remote_addr":"[fd00:1122:3344:101::b]:63452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.450113604Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.450146855Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"be84e373-3a25-4f16-ac1a-160f6178f116","remote_addr":"[fd00:1122:3344:101::b]:63452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.459928833Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"da9dea34-a9a0-47c9-a1dd-28693c048d38","remote_addr":"[fd00:1122:3344:101::b]:63452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.504145386Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.504303548Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"41a2e3c9-92ac-42ef-9aaa-11fe2b979fb2","remote_addr":"[fd00:1122:3344:101::b]:61131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.542386355Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50025"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.542454446Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.583255611Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.643456209Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d8fffb97-96a4-438f-a17c-81ff5a7bcaf0","remote_addr":"[fd00:1122:3344:101::b]:50025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101013,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.643601491Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.643773093Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"13f486e0-c63e-4eba-9283-248886a10128","remote_addr":"[fd00:1122:3344:101::c]:33630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.643916506Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e48834d9-fb75-4714-afae-40643bfdeb4e","remote_addr":"[fd00:1122:3344:101::b]:50025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.6591258Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.65915419Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e2e64d23-fbb0-48d3-ac31-3174a3159f3f","remote_addr":"[fd00:1122:3344:101::c]:33630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.667143343Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8148fa99-f1b6-4801-b19d-0a955de42250","remote_addr":"[fd00:1122:3344:101::c]:33630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691210482Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58026"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691370724Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691400275Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691424455Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691432815Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691439255Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691444485Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691454645Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.691470025Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"eb28f09e-f5e7-4809-bb14-3dc96ab330da","remote_addr":"[fd00:1122:3344:101::b]:58026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.825481714Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.825615906Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"aad8ca42-ecbf-49dd-8814-578d891081fd","remote_addr":"[fd00:1122:3344:101::c]:48769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.868892376Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60671"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869052848Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869084499Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869095949Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869108959Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869117309Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869125289Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869135509Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.869152029Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"194176d3-951d-4838-a697-23b1bb07fb62","remote_addr":"[fd00:1122:3344:101::c]:60671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.914040812Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.914094633Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.950166191Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.998455972Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0446e6d8-8651-45ad-9d59-b33045d2b5c3","remote_addr":"[fd00:1122:3344:101::c]:54177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84382,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:11.998695975Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"938e0249-3752-4735-8ed8-49b4a8d26aa3","remote_addr":"[fd00:1122:3344:101::c]:54177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:28.319351155Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:28.319555547Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"8868a1a6-035f-4e09-9af7-cb4920dee068","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43310","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.569384301Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33183"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.569508863Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c83735f2-4e63-48a2-b268-f5e7eb3cd99b","remote_addr":"[fd00:1122:3344:101::a]:33183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.618894149Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.619095262Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9170209d-2119-44c5-ab7c-5935a0dc6830","remote_addr":"[fd00:1122:3344:101::a]:44181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.661596341Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.661622081Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d7b28b90-9788-4290-a3ce-8e74cf288b20","remote_addr":"[fd00:1122:3344:101::a]:33183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.671649302Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3e1a3497-a4c9-4a55-b3e8-c0852a929886","remote_addr":"[fd00:1122:3344:101::a]:33183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.717917774Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52433"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.717995785Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.754778584Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.80771248Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5175d9d8-f4b9-4da5-9c3b-5f6fcd930c33","remote_addr":"[fd00:1122:3344:101::a]:52433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89750,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:40.807964113Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c7d2564f-57ab-43d6-8e85-67eceb87da91","remote_addr":"[fd00:1122:3344:101::a]:52433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.364018549Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.364129621Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1bacd4b4-f511-4af3-a8bb-04ae03ba96b6","remote_addr":"[fd00:1122:3344:101::b]:41615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.408556847Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37577"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.40880701Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"4ce6f9bd-385a-48ce-9269-4b8f60408634","remote_addr":"[fd00:1122:3344:101::b]:37577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.409693532Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.409723653Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8a65b12a-66ab-4cff-8cf9-e566de445634","remote_addr":"[fd00:1122:3344:101::b]:41615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.417763886Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"de853b41-7016-4ee8-b22e-a04a23497f36","remote_addr":"[fd00:1122:3344:101::b]:41615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.433549479Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52199"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.43364379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.479877002Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.542161519Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5815f6c3-6881-481e-94a2-e937ad6b8660","remote_addr":"[fd00:1122:3344:101::b]:52199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108534,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.542306841Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49524"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.542466414Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3efdac56-1a17-41e4-800f-a8f6edc68198","remote_addr":"[fd00:1122:3344:101::c]:49524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.542657286Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8aa5bb1a-631f-4e5e-a658-7c17a275f600","remote_addr":"[fd00:1122:3344:101::b]:52199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.587497158Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.587709461Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7966e395-2def-45dd-90b2-decb4378c4bb","remote_addr":"[fd00:1122:3344:101::c]:53838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.639610523Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.639630113Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"82be8c03-fca4-4f45-9d4c-9bacdd11ec06","remote_addr":"[fd00:1122:3344:101::c]:49524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.642882579Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55378"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.64296893Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.681615735Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.739920046Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dd19c06a-f6a0-4b6e-8fd0-ed92eae3fe21","remote_addr":"[fd00:1122:3344:101::c]:55378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96974,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.740335392Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"78c16633-9074-4082-b265-98c33d5ca2d8","remote_addr":"[fd00:1122:3344:101::c]:55378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":197,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:41.740368943Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cdbd5863-82c7-4671-afa1-e453faabccb9","remote_addr":"[fd00:1122:3344:101::c]:49524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:58.352609361Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:22:58.352742523Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c9c58224-2211-4916-83ae-e01d8fad5057","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34836","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.779281856Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.779413098Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"207067a4-7884-4a83-8e7b-30d7579f2551","remote_addr":"[fd00:1122:3344:101::a]:57993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.833356818Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.833388608Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e5566719-959c-4f96-931f-efa91993dc59","remote_addr":"[fd00:1122:3344:101::a]:57993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.842819921Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e8e9e1fb-f4e0-43ff-b45e-3fd66db2c854","remote_addr":"[fd00:1122:3344:101::a]:57993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.908091231Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.908276284Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b092408d-8175-4151-ab44-453489db897f","remote_addr":"[fd00:1122:3344:101::a]:39474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.935858032Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51168"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.935929804Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:10.978511654Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.040313524Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fb73f39e-d780-469c-996c-90ba189bb3d8","remote_addr":"[fd00:1122:3344:101::a]:51168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104405,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.04068107Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b7e99d4e-cf8c-400b-bb3f-3921bec024df","remote_addr":"[fd00:1122:3344:101::a]:51168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.235393194Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.235506745Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e3b49c1d-97fc-4643-87a7-9867ef65d495","remote_addr":"[fd00:1122:3344:101::b]:41650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.299291594Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.299325804Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a477b213-12b5-433d-9a3f-ffe71845cfb4","remote_addr":"[fd00:1122:3344:101::b]:41650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.31320435Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"73970ad8-92ae-4fd5-8203-c00535d7fd55","remote_addr":"[fd00:1122:3344:101::b]:41650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.552926308Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52318"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.553134931Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b708ea10-6fd9-4ee7-8ce1-60eed05b9100","remote_addr":"[fd00:1122:3344:101::b]:52318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.61836909Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47478"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.618414521Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.658716829Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.716894849Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6b18de64-f286-457a-b7ab-eab561c4509e","remote_addr":"[fd00:1122:3344:101::b]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98498,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.717133132Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.717401506Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7e5bbd6f-3c4e-4e3d-9122-dfec947ee9ad","remote_addr":"[fd00:1122:3344:101::c]:47398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.717437406Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"86f3e85a-05a1-471f-9117-61e2611bc696","remote_addr":"[fd00:1122:3344:101::b]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":249,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.761810771Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.761981024Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"1eb48f4a-f597-414c-b745-a6c01f3a8dc3","remote_addr":"[fd00:1122:3344:101::c]:56117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.803162204Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.803217225Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"332bd8a6-e00e-4bfb-a779-5f08acaf0e3f","remote_addr":"[fd00:1122:3344:101::c]:47398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.811498332Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2b8cccbd-9e8e-4421-bda3-90bb3f672b3e","remote_addr":"[fd00:1122:3344:101::c]:47398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.84047527Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38969"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.840555571Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.880639666Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.938729425Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3b186b15-d56b-4281-8e0c-c0c74ac6ac68","remote_addr":"[fd00:1122:3344:101::c]:38969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98197,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:11.939015049Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5ea9a9d7-b478-49f3-ad88-13335c28d42e","remote_addr":"[fd00:1122:3344:101::c]:38969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:22.061338942Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:22.062166183Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:22.100002767Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999421s","next_cleanup":"Instant { tv_sec: 1980, tv_nsec: 997974265 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:28.386214991Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:28.386339353Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f3656918-9e25-4a34-a48a-ce5fe1ff024f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40493","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:38.54440207Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.551022337Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46576"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.551142739Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c424960e-75dc-4f99-8b7b-a9cff98d0502","remote_addr":"[fd00:1122:3344:101::a]:46576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.565624153Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.565639613Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b6e9011e-8943-4586-90fe-ba11309927d2","remote_addr":"[fd00:1122:3344:101::a]:46576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.635569988Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.635794601Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7b18db8e-9699-4fc6-97bc-5f8c5a14f793","remote_addr":"[fd00:1122:3344:101::a]:46248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.638892325Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"64df8cd4-8465-4892-841e-85b9c42e747b","remote_addr":"[fd00:1122:3344:101::a]:46576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.711442267Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65298"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.711532599Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.749554425Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.803201711Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"53d18c61-98cf-4110-9d31-f7584b508c7c","remote_addr":"[fd00:1122:3344:101::a]:65298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91711,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:40.803470014Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"754e26ee-8882-4d37-bcdb-783a1cc971ab","remote_addr":"[fd00:1122:3344:101::a]:65298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.16791065Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.167998931Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"03a5fc2b-d457-4203-8f2d-b1071f01e2ff","remote_addr":"[fd00:1122:3344:101::b]:50127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.214976703Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.215011094Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"263bb77a-ee45-4f98-b069-91fd6e3dfa7c","remote_addr":"[fd00:1122:3344:101::b]:50127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.22468584Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cd2456f8-541b-4145-91fb-82eac9c656db","remote_addr":"[fd00:1122:3344:101::b]:50127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.323677445Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.323862717Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"4fe9f64f-0230-451e-8227-ae7af1f2d1be","remote_addr":"[fd00:1122:3344:101::b]:52658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.339398466Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:32773"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.339510348Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.388308766Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.462570072Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"01ba53a1-575f-42c2-9bff-fd6ae5446d95","remote_addr":"[fd00:1122:3344:101::b]:32773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.463074559Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"601d8b70-fcc1-4f2f-899d-796c0d4e926a","remote_addr":"[fd00:1122:3344:101::b]:32773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.549939743Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.550069415Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9346e80b-6c99-42d5-be2a-f3d3544190ef","remote_addr":"[fd00:1122:3344:101::c]:62773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.60579501Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.605856041Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"44f7de70-a370-45eb-a516-927dcfd617df","remote_addr":"[fd00:1122:3344:101::c]:62773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.615891963Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cd40ce07-f67e-4591-bc2b-2dd51acebce9","remote_addr":"[fd00:1122:3344:101::c]:62773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.637355905Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40951"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.637430096Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.676325784Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.73770922Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5c6833b6-7642-4163-8008-11c44f2c43a9","remote_addr":"[fd00:1122:3344:101::c]:40951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100300,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.737830801Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41919"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.738110985Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"81d6d2bd-99d2-443f-aad4-ff7890c8f4e1","remote_addr":"[fd00:1122:3344:101::c]:41919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":181,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:41.738210207Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6b5f8028-9109-4905-b7ab-2160bbd0655a","remote_addr":"[fd00:1122:3344:101::c]:40951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:58.419530979Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:23:58.419725331Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9f0298bf-a25e-4a81-8a5b-7535ff4b9f3e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47458","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.677440636Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.677557448Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"194cdd4d-e010-45f4-a8d3-a072ab46d760","remote_addr":"[fd00:1122:3344:101::a]:42617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.723748679Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.723794289Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"baff9345-7d66-4d19-a1cc-67c17b30e7e9","remote_addr":"[fd00:1122:3344:101::a]:42617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.732310779Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"325e8cc9-8f39-4b48-b3a8-089eaabbf272","remote_addr":"[fd00:1122:3344:101::a]:42617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.845406663Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:10.845564665Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b6dd8aff-5e9b-436e-968d-274880d070c0","remote_addr":"[fd00:1122:3344:101::a]:47367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.037091994Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42478"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.037163786Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.087535615Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.165505494Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ceea25b0-3f82-41fc-ba9b-a86d92e62767","remote_addr":"[fd00:1122:3344:101::a]:42478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128364,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.165861319Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"99e96db8-6c40-4a4a-9fde-a480b051aea3","remote_addr":"[fd00:1122:3344:101::a]:42478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.412284461Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.412450864Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"28d3344c-5ffe-431e-8fd8-f89f5a88e25b","remote_addr":"[fd00:1122:3344:101::b]:55133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.480714756Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.480748946Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"daeb2351-87dc-450f-ab51-064e4153d130","remote_addr":"[fd00:1122:3344:101::b]:55133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.49163094Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"51f1bd2b-3de5-4bd8-b2ad-66a1c2283050","remote_addr":"[fd00:1122:3344:101::b]:55133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.543962957Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59481"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.544078279Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.587872216Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.646210438Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ff9cd3cb-995e-48f1-994d-3d153770d722","remote_addr":"[fd00:1122:3344:101::b]:59481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.64633742Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.646691575Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"a7a0fb26-a7c8-4397-86d5-9f7830e0fbec","remote_addr":"[fd00:1122:3344:101::b]:41915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.646716675Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e4ac2685-87e9-4bc8-bc2b-b9ee36077107","remote_addr":"[fd00:1122:3344:101::b]:59481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.717240489Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.717362731Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3ba70919-c2f6-4964-956d-8a530b5e85fe","remote_addr":"[fd00:1122:3344:101::c]:59863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.738269945Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.738287615Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c3f815a5-8781-4d48-86be-c42d9db8646a","remote_addr":"[fd00:1122:3344:101::c]:59863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.755250725Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2060a6ae-0aaf-4ea3-b74c-bf9ba96e74b8","remote_addr":"[fd00:1122:3344:101::c]:59863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.831763853Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.831912765Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6d419847-332f-4982-b594-53644f44af18","remote_addr":"[fd00:1122:3344:101::c]:48593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.844726526Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54594"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.844811017Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.884302423Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.945902211Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"759bf82a-23a5-47b1-a52f-4f609d432636","remote_addr":"[fd00:1122:3344:101::c]:54594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101111,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:11.946219246Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"be813143-053d-455c-b9c3-43b1ba6924f6","remote_addr":"[fd00:1122:3344:101::c]:54594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.488046276Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.490568232Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.491239221Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.491926111Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.492601771Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.493729896Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.494800562Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.495674704Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:20.496355784Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:28.453106909Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42857"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:28.453256901Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a6a01a5e-629c-4c73-96d2-3451a4f635b2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42857","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.453609481Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.454928379Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.455640589Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.477921803Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.488838567Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.490018563Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.490705443Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.491364602Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.49188509Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.492545359Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.493049466Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.494144152Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.494635519Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.495145176Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.495790245Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.496702368Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.49825455Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:30.498746086Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.456599102Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.457978991Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.458711892Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.48059087Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.490835984Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.491497863Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.493790956Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.494449105Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.494963542Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.495617202Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.496108659Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.496609035Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.497119863Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.49762536Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.498272369Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.499175452Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.499712369Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:40.500196726Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.546022392Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62008"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.546148914Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f5b0bd37-8c68-4f79-8ee2-d0a806fb6ed3","remote_addr":"[fd00:1122:3344:101::a]:62008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.559906118Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.559938358Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"025e4b13-b356-44c6-bb0b-edd7d47231c3","remote_addr":"[fd00:1122:3344:101::a]:62008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.56859554Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5e442d60-40df-4709-94bd-cbe7a2ff1b25","remote_addr":"[fd00:1122:3344:101::a]:62008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.626221192Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53026"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.626297153Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.666179245Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.719061741Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"214c1743-1e6b-4f0e-ab87-14531ca8e903","remote_addr":"[fd00:1122:3344:101::a]:53026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92797,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.719236123Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.719552857Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5525d99c-dbba-4357-92fc-0ac81bb5d0e9","remote_addr":"[fd00:1122:3344:101::a]:50316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":219,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:40.719596558Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3dc53992-b357-4320-a4ae-4077634af142","remote_addr":"[fd00:1122:3344:101::a]:53026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":277,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.145060984Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44461"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.145179535Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"66aeca54-ac83-4a5b-b1bb-1ba5a49ca5fc","remote_addr":"[fd00:1122:3344:101::b]:44461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.189966086Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.189999837Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ca6553a0-e91d-4603-a2cd-91a80479f3f8","remote_addr":"[fd00:1122:3344:101::b]:44461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.202787717Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"266f2ddc-39ca-4d1c-b0f3-509e0967b429","remote_addr":"[fd00:1122:3344:101::b]:44461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.309845966Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48159"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.309936637Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.370197426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.431252847Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b56bffd0-3d9a-4212-ae6d-2ce5931a22ad","remote_addr":"[fd00:1122:3344:101::b]:48159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121356,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.43149406Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.431602512Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"bb828d73-3a41-47e3-a11d-aa89a2fa268e","remote_addr":"[fd00:1122:3344:101::b]:48159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.431812615Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"fe04b83a-542f-467a-bdda-a7747ae67a3b","remote_addr":"[fd00:1122:3344:101::b]:60472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.448619886Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46782"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.448764413Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9ced5be7-7988-43bb-bf7b-791f4ad85f55","remote_addr":"[fd00:1122:3344:101::c]:46782","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.513164131Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.513196472Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"38c4b6de-6440-4940-aeef-683c294288e4","remote_addr":"[fd00:1122:3344:101::c]:46782","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.526825913Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a8d2f1c6-5fb2-4d92-afc2-306401950371","remote_addr":"[fd00:1122:3344:101::c]:46782","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.628881662Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63203"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.628986003Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.670347666Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.724256976Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"47728576-dc93-453b-a7ef-59c4881e1d74","remote_addr":"[fd00:1122:3344:101::c]:63203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.724435478Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.724678882Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7c1fee12-afac-4ec8-89b1-481acb1c00c7","remote_addr":"[fd00:1122:3344:101::c]:46781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:41.724704022Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6fb7e923-10d0-48c1-92eb-1e7a15b19d82","remote_addr":"[fd00:1122:3344:101::c]:63203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.459032356Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.460382724Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.461067064Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.481869757Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.493121236Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.49480091Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.495490889Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.496129168Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.496658636Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.497310105Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.497791062Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.498291699Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.498762365Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.499244652Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.499886021Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.500810904Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.501301331Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:24:50.501795948Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:58.484705463Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:24:58.484826274Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c3e92ea5-b4a9-4e6b-885b-2aa0a2faff3b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62179","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.46153396Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.462742017Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.463466117Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.48424474Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.495943765Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.496660355Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.497331095Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.497982514Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.49847186Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.4991165Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.499576256Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.500052313Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.500512299Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.500990596Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.501618215Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.502500277Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.502978084Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:00.503448331Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.46371999Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.465041919Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.465798019Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.486697764Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.498207266Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.498907266Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.499596446Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.500235105Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.500743402Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.501443912Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.501946529Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.502471826Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.503235967Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.503942337Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.504777049Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.506064847Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.507322855Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:10.507891923Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.698420828Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.698552879Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"014dbe69-0055-4156-8c53-3aa4b2a25db1","remote_addr":"[fd00:1122:3344:101::a]:45697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.721064567Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57050"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.721254859Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c1bac4c8-9a81-4d0a-9da0-eec5c0e41f5c","remote_addr":"[fd00:1122:3344:101::a]:57050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.735666593Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.735682373Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e101bdf7-0c6a-4621-88d0-727cbd5b404c","remote_addr":"[fd00:1122:3344:101::a]:45697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.743519253Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c11001c0-1455-4340-a144-622d324bc506","remote_addr":"[fd00:1122:3344:101::a]:45697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.875757867Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65248"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.875827538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.919018066Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.978306592Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fd2e7048-3a13-4a9b-bce3-d1a723ad7e33","remote_addr":"[fd00:1122:3344:101::a]:65248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102503,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:10.981115121Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"66d188ab-83eb-4f14-ab38-be8d564aadde","remote_addr":"[fd00:1122:3344:101::a]:65248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.379937771Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.380050813Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9916c173-9603-4fd9-bb50-0e0b3335def2","remote_addr":"[fd00:1122:3344:101::b]:33453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.421133032Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.421352565Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7d0c3184-5213-4fab-b8e9-12ad5c01d05c","remote_addr":"[fd00:1122:3344:101::b]:56284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.452647266Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.452679897Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f9883349-4069-46c4-8198-d2e59afd856a","remote_addr":"[fd00:1122:3344:101::b]:33453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.471441611Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"43a3f189-6c56-4752-9e50-8cc8dce423c7","remote_addr":"[fd00:1122:3344:101::b]:33453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.526952963Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44384"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.527044974Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.566900196Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.625215668Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b4baf0c5-4a60-41fb-8f7e-d726eaa80122","remote_addr":"[fd00:1122:3344:101::b]:44384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98214,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.625536722Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4b4e7fb6-f80c-40ef-b225-4af7a89587d4","remote_addr":"[fd00:1122:3344:101::b]:44384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.663992414Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.664116786Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c4c4474f-8304-4c18-916d-ab8aeb283e74","remote_addr":"[fd00:1122:3344:101::c]:62886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.704644847Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.704666737Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e27c5528-72ca-4684-bb6b-c41d16668434","remote_addr":"[fd00:1122:3344:101::c]:62886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.716320002Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.716512064Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"86fd2de0-60fc-4292-90b4-c5e6a3ca63fc","remote_addr":"[fd00:1122:3344:101::c]:58216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.72545585Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"86e04f25-f39c-403b-944f-3862c26e9687","remote_addr":"[fd00:1122:3344:101::c]:62886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.891654972Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36757"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.891739164Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.930033233Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.989249968Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"65e9f88a-a20a-4340-a4f6-63d4fc962b44","remote_addr":"[fd00:1122:3344:101::c]:36757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97548,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:11.989504802Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"eef7aab5-e99a-4476-85d1-6563c854909b","remote_addr":"[fd00:1122:3344:101::c]:36757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.465243421Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.467474862Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.468207303Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.488056232Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.500482598Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.501184837Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.501847517Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.502487676Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.502988853Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.503660463Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.504139909Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.504635786Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.505132783Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.50563621Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.506251409Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.508178986Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.508661483Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:20.509787359Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:28.517523563Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57012"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:28.517650855Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5a485dac-7aee-4aa3-8233-c47f54cd6c33","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57012","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.467187318Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.469439089Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.470260751Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.49006621Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.50280647Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.503701832Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.504370112Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.50499258Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.505497468Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.506133746Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.506630584Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.50711647Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.507617477Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.508098764Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.508724513Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.51063471Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.511133257Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:30.511647294Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.46596681Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59780"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.466082031Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"90b6d861-b56a-460c-81cb-58dc516775c1","remote_addr":"[fd00:1122:3344:101::a]:59780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.469951696Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.471587099Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.472439251Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.492843998Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.505935803Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.506682953Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.507377333Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.508028532Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.50857529Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.50925492Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.509769957Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.510275264Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.510775381Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.511274058Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.511894367Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.513007303Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.51350371Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:40.514018877Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.51851953Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.518533631Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"29e22acb-07b9-473d-a94e-e548c991efaf","remote_addr":"[fd00:1122:3344:101::a]:59780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.528305818Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7172a916-1f5d-4da9-a394-da7a04c00a04","remote_addr":"[fd00:1122:3344:101::a]:59780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.598767871Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59605"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.598877273Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.641582905Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.705155921Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"164578ca-6bb4-4250-a8df-682ac784b5b8","remote_addr":"[fd00:1122:3344:101::a]:59605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106320,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.705388354Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56792"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.705686938Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cf06adaf-ccad-4059-acfd-13210af4532e","remote_addr":"[fd00:1122:3344:101::a]:59605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":249,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:40.70582631Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f90be76f-e390-4ea3-8240-17bca94a5e71","remote_addr":"[fd00:1122:3344:101::a]:56792","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":275,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.266884326Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38541"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.267004748Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8c8298e0-9071-4b7a-b420-b64a90409f17","remote_addr":"[fd00:1122:3344:101::b]:38541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.310092835Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.310161186Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c7e2188f-13e2-472f-a95c-ab1b256cb7c1","remote_addr":"[fd00:1122:3344:101::b]:38541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.342200678Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.342423031Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f60b8d47-de78-4b9d-a9dc-feb39e750e22","remote_addr":"[fd00:1122:3344:101::b]:36283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.351152994Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"64844f20-52a2-40fe-8bac-9fe18cd6cff8","remote_addr":"[fd00:1122:3344:101::b]:38541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.412491568Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47894"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.41260554Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.456429888Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.521827689Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1e2e9a40-e90f-49cd-bc48-8743fcfd92f4","remote_addr":"[fd00:1122:3344:101::b]:47894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109272,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.522162474Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c2e6507e-5241-4a42-befa-733aa1f7fca2","remote_addr":"[fd00:1122:3344:101::b]:47894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.609593076Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37146"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.609710208Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8ba4d2ab-7fe7-4834-a784-73680ab6b61b","remote_addr":"[fd00:1122:3344:101::c]:37146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.649759302Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.649778342Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"cc15754f-a9cf-4a8b-95d5-dc1376ded705","remote_addr":"[fd00:1122:3344:101::c]:37146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.657891107Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7b78fceb-55e4-4536-80a2-99750e70a159","remote_addr":"[fd00:1122:3344:101::c]:37146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.665024757Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52989"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.665125269Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.710717101Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.774280987Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fc66ea89-7485-4ba9-9243-f755e4573585","remote_addr":"[fd00:1122:3344:101::c]:52989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.774456539Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.774715793Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6bf2f917-2566-4931-8f9e-745dba3a3957","remote_addr":"[fd00:1122:3344:101::c]:62343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:41.774782154Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9cec4dfd-d427-44b3-885b-8e1125fadb9c","remote_addr":"[fd00:1122:3344:101::c]:52989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.472029155Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.473322383Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.474019052Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.494913407Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.512570006Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.513329626Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.514057357Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.514738956Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.515270484Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.515926783Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.51642454Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.517075029Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.517551496Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.518071183Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.518722182Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.519780948Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.520277384Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:25:50.520935464Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:58.550250513Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:25:58.550444675Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4284082b-80dc-44e3-919e-674d891e4d6e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46170","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.474892184Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.476213933Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.476951523Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.497032616Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.514619784Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.515361774Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.516060594Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.516723874Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.517264881Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.517933341Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.518441798Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.518974485Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.519501063Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.52001281Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.520668419Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.521616833Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.52214968Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:00.522684568Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.47679008Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.478588825Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.479721641Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.498966902Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.517150088Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.517949829Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.518722411Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.51943253Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.520008478Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.520740489Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.521261976Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.521830814Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.522408932Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.5229595Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.52369569Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.524712185Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.525266093Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:10.525830571Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.725671467Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.725854149Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"03282c23-b15e-46fd-a6b8-2285c5dea685","remote_addr":"[fd00:1122:3344:101::a]:47763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.786251781Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.786277571Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"070bb101-c67a-4ef7-9494-a7464beb1301","remote_addr":"[fd00:1122:3344:101::a]:47763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.793864138Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.794153052Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"8f106366-5ee6-474e-8ced-a5af1632318e","remote_addr":"[fd00:1122:3344:101::a]:56396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.806709159Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"70179cdf-1354-41d6-9609-d2cacaf4562f","remote_addr":"[fd00:1122:3344:101::a]:47763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.908419012Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55697"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.908511213Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:10.967031438Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.029806713Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7271224b-6650-4a12-9c7c-6be405e991d1","remote_addr":"[fd00:1122:3344:101::a]:55697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121342,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.030410011Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"31bccfc6-4076-4666-aed6-221c90c161e3","remote_addr":"[fd00:1122:3344:101::a]:55697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.392036097Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39277"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.39223673Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7b0566d0-149e-4f51-a6a7-8c2127a87579","remote_addr":"[fd00:1122:3344:101::b]:39277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.461534846Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.461616267Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7fba1af3-be2f-46a3-8acc-29f89f0c1f76","remote_addr":"[fd00:1122:3344:101::b]:46240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.477201107Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.477218597Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2c3fe8a7-210e-4da4-9c62-ade0db95fa7e","remote_addr":"[fd00:1122:3344:101::b]:46240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.486080422Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0e8f8322-8c63-4afa-a3a5-fbf1d9fcd369","remote_addr":"[fd00:1122:3344:101::b]:46240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.553877687Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.553966689Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ecfa753a-1afd-4a17-abe7-cf22de539ecf","remote_addr":"[fd00:1122:3344:101::c]:49449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.600684567Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36479"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.600805339Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.645798063Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.708679939Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a1c12f9a-09cd-4de0-9316-218bdf9d4e8d","remote_addr":"[fd00:1122:3344:101::b]:36479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107916,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.708868892Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.708886172Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0122dad0-d707-4fa7-930c-67c83f73e9ee","remote_addr":"[fd00:1122:3344:101::c]:49449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.709026164Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"26c539c1-a641-4f0b-a702-24f1dc94b75f","remote_addr":"[fd00:1122:3344:101::b]:36479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.903488654Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.903656277Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"cbfe9c08-5bd1-4172-a4a8-f8398f296485","remote_addr":"[fd00:1122:3344:101::c]:37630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:11.910575964Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"92bc7834-b916-4b83-bf07-4d2ed82ada77","remote_addr":"[fd00:1122:3344:101::c]:49449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:12.01815188Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51082"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:12.018238551Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:12.062776619Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:12.122511171Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"33001277-bd9a-4fc9-bc88-81b526082b32","remote_addr":"[fd00:1122:3344:101::c]:51082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104298,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:12.122947487Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d4689236-7288-4969-85b1-c2b1b4d0105e","remote_addr":"[fd00:1122:3344:101::c]:51082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.479210093Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.480764795Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.481472095Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.500724226Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.519978017Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.520677767Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.521372917Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.522008646Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.522556854Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.523227873Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.52370162Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.524220377Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.524721134Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.525205751Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.52583578Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.526760143Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.52724847Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:20.527727637Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:28.583127102Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:28.583310644Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"152ff884-e1a9-4140-9194-9ed34fbc2e38","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54216","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.481445023Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.482713821Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.483413791Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.502220345Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.521500427Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.522193247Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.523920771Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.524577731Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.525148309Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.525809618Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.526333755Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.526861493Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.52739786Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.527934758Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.528586417Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.529537141Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.530052368Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:30.530564015Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.483759324Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.485064703Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.485755403Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.503833858Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.52318078Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.52385889Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.525574714Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.526241223Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.526820181Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.52747374Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.527976718Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.528506425Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.529017882Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.52952682Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.530148858Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.531077411Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.531556798Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:40.532067695Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.559817466Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.559935028Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b05845cb-e6f7-4ef9-9499-ae254810af59","remote_addr":"[fd00:1122:3344:101::a]:50256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":41,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.602727351Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.602753652Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4bf0e602-29be-4d35-bc08-84444cb1fe14","remote_addr":"[fd00:1122:3344:101::a]:50256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.611540915Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7448975e-2d01-4b20-aca6-c3f2eff3dce9","remote_addr":"[fd00:1122:3344:101::a]:50256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.644775124Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38092"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.644856305Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.683823974Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.745476203Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"016d709e-fc91-4158-a3fa-90d55b2dae24","remote_addr":"[fd00:1122:3344:101::a]:38092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100661,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.745726796Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.74599059Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9b51d542-cbbf-4297-a052-2da3b0dbc371","remote_addr":"[fd00:1122:3344:101::a]:38092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:40.746143232Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"fd93ff60-7cca-42be-a0d1-95c500e1ab41","remote_addr":"[fd00:1122:3344:101::a]:46055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.200422574Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.200625027Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5f390f7a-cf9e-4b1e-b605-9f8f28448733","remote_addr":"[fd00:1122:3344:101::b]:33669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.251388002Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33542"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.251473223Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7d8915c7-2ee6-4a82-a823-def6a201ed6c","remote_addr":"[fd00:1122:3344:101::b]:33542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.265857186Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.265873286Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"102b5893-632a-46ae-97f2-3c04c677ab64","remote_addr":"[fd00:1122:3344:101::b]:33542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.275110526Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4d033f79-c585-4f77-afd5-0ba39c08be0c","remote_addr":"[fd00:1122:3344:101::b]:33542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.432736797Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41525"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.432823129Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.47478347Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.534667984Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ceaa9abe-1237-4952-80f4-3cef9de88b12","remote_addr":"[fd00:1122:3344:101::b]:41525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101885,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.534916757Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56741"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.535018129Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.535219572Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"28576a70-16f2-4a57-a298-13a3a479a5e3","remote_addr":"[fd00:1122:3344:101::b]:41525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":289,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.535309153Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"578a8aac-3710-4f89-91c5-2d5c4f5df4e6","remote_addr":"[fd00:1122:3344:101::c]:55510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.535466565Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"10a5bf1b-947d-4103-ba3f-6fe82b566542","remote_addr":"[fd00:1122:3344:101::c]:56741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":244,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.578097596Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.578116846Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"67e23d4c-a733-4b85-bf27-5ddc874a502f","remote_addr":"[fd00:1122:3344:101::c]:55510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.587780422Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a2a7c41c-bd8f-4e50-a6af-2e2b281af89c","remote_addr":"[fd00:1122:3344:101::c]:55510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.600840806Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55421"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.600939758Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.643787182Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.708622235Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"11ebe588-f5ca-4ba2-93ea-6f5ba3fcf8e7","remote_addr":"[fd00:1122:3344:101::c]:55421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107700,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:41.708923969Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8f3394ec-49d7-441d-b266-224cbe92ccb8","remote_addr":"[fd00:1122:3344:101::c]:55421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.485514689Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.486755476Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.487482296Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.505318378Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.524719921Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.525471072Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.527151425Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.527780164Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.528298811Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.528932111Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.529409867Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.530918988Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.531396565Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.531911773Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.532563092Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.533491565Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.533969602Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:26:50.534456528Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:58.616102514Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:26:58.616253497Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"ce0f604d-aecb-4086-b0cd-b5fbb9b4a4b0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46522","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.487586617Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.488787404Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.489471624Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.506198139Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.527766053Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.528465303Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.529147503Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.529764531Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.530253558Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.530893398Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.531349744Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.532863305Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.533327632Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.533829599Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.534458018Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.53532834Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.535952389Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:00.536447796Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.490310095Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.491560612Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.492292523Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.508157817Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.529900693Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.530598023Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.531277272Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.531899911Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.532421278Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.533065068Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.533588755Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.535120336Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.535626384Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.536136591Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.536761069Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.537661532Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.538147279Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:10.538641026Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.849257993Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.849385165Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"68b59697-00e9-42f8-abe7-f65859c906a4","remote_addr":"[fd00:1122:3344:101::a]:60835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.880177679Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49645"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.880351041Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f40cd599-6a26-49e6-866b-3704fbd6988b","remote_addr":"[fd00:1122:3344:101::a]:49645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.928571831Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.928599391Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"bf7c3bbc-b95b-496b-ac64-bbaa64f41b5c","remote_addr":"[fd00:1122:3344:101::a]:60835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.977320098Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39458"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.977420109Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.977435479Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.97745546Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.978004878Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.987176517Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"01651b03-5e5c-4623-a9c1-e35270da0597","remote_addr":"[fd00:1122:3344:101::a]:60835","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":506,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.994901256Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:10.994923316Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.026913377Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.026973938Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.05414586Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58019"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.054553276Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.068256429Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.068346291Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.118166853Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.118265454Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.125977583Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.156648075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.156721036Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_fad2b250-7ced-4cac-87db-22998050be7c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.187442799Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.1875428Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f4bc5b75-b581-474b-87db-711856d8624e"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.211441867Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37202"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.211996945Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212072186Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212207568Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212263359Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212283929Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.21236649Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212416901Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.212459382Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"64ec7a19-1acb-4759-b514-4c81b761992d","remote_addr":"[fd00:1122:3344:101::a]:37202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":817,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.21451772Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"30e794d5-4dad-4019-accb-13eb95bfa1fa","remote_addr":"[fd00:1122:3344:101::a]:58019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160140,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.214861465Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"410f1c5a-f465-49e1-a16f-2a082c408a90","remote_addr":"[fd00:1122:3344:101::a]:58019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.220187951Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.220211391Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.250104692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.250163633Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.277795452Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.277857173Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.290032224Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"c4ffe23d-532f-4b18-af8e-89fd77674468","remote_addr":"[fd00:1122:3344:101::a]:39458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":312651,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.45078802Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.450914832Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a86431c2-2034-4ffb-bd46-1f255ecd0a2b","remote_addr":"[fd00:1122:3344:101::b]:57946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.494203462Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.494277333Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ddc1e41e-4e36-44d2-b0ec-10d0ec9d4c6f","remote_addr":"[fd00:1122:3344:101::b]:57946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.506301762Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"335510c6-401e-4530-abbb-64fa3997ff9b","remote_addr":"[fd00:1122:3344:101::b]:57946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.647665904Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59618"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.647745936Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d6b07ea3-36a2-4694-95cb-186b11718341","remote_addr":"[fd00:1122:3344:101::c]:59618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.661417858Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.661620711Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7ca08def-ea27-42a8-9a88-04bb861d4fe4","remote_addr":"[fd00:1122:3344:101::b]:50939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.663288344Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35763"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.663419006Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.663434217Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.663476917Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.663867693Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.664090636Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37976"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.670531586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.681707354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.681778775Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.718876848Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.718956349Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.72683169Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.742729354Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42630"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.742828525Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.742835976Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c64a3854-cad1-4d17-adb2-3f6f4e743c5a","remote_addr":"[fd00:1122:3344:101::c]:59618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":678,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.752415011Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fca21f0e-7af4-4e74-9508-4c3ae81ea2a3","remote_addr":"[fd00:1122:3344:101::c]:59618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":381,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.753912592Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.753930382Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.78782243Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.78788939Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.792386244Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64986"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.792472035Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.792977082Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"137f8c06-7342-46d5-a9bc-53a7c72cc7ef","remote_addr":"[fd00:1122:3344:101::c]:64986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":343,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.793330107Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.793408158Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.793478289Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.79350287Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.79351307Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.7935402Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.79355296Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.793576571Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c2879589-a5ab-4f92-bfad-13d3cc821d9f","remote_addr":"[fd00:1122:3344:101::b]:51452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":687,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.808952837Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.809026048Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8db267ab-eb21-4d8e-a2b1-993085d28122","remote_addr":"[fd00:1122:3344:101::b]:37976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138762,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.813651213Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5d4d6c9c-fc79-4568-b9a8-6eab0927f985","remote_addr":"[fd00:1122:3344:101::b]:37976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":757,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.816399172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.816427982Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_fad2b250-7ced-4cac-87db-22998050be7c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.842588591Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.842660922Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f4bc5b75-b581-474b-87db-711856d8624e"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.849909644Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.854915705Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41282"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.856985234Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.857004684Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.857063625Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.863240382Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.871344247Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.871723572Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db6f4d38-f0e6-466e-b782-b9022f769535 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.888486038Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.888542129Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db6f4d38-f0e6-466e-b782-b9022f769535"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.893870054Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64618"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894401651Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4d6538e9-2779-4ffe-b732-b8ef5b68fff3, kind: Service { id: 50ef7a5a-8210-43f1-a548-846251e25074 }, name: Name(\"external-dns-50ef7a5a-8210-43f1-a548-846251e25074\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 88])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 90d25e95-e0a0-4767-914f-6a42c8b1e264, kind: Service { id: 403cce04-ffba-4276-94a6-76cab9338e71 }, name: Name(\"external-dns-403cce04-ffba-4276-94a6-76cab9338e71\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 142])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 63031233-d4cf-486c-a883-b94591ef6215, kind: Service { id: bbffc4db-58c5-4210-973d-49d52a55513e }, name: Name(\"nexus-bbffc4db-58c5-4210-973d-49d52a55513e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 9])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab6eb7eb-9f19-4410-8b07-657b90ce790a, kind: Service { id: 5382fba6-edea-49d2-8ecc-9b1edfc688be }, name: Name(\"nexus-5382fba6-edea-49d2-8ecc-9b1edfc688be\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e349c246-31d7-4a6d-8e38-cf8679bf8d63, kind: Service { id: 3285ddd9-4901-4d45-8459-dd50ba0635ef }, name: Name(\"nexus-3285ddd9-4901-4d45-8459-dd50ba0635ef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 227, 174])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894456922Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894513513Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894549894Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894564874Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894589094Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894619695Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.894648615Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"86961892-03b9-4247-8bca-7937936e4f39","remote_addr":"[fd00:1122:3344:101::c]:64618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":570,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3be44b5b-5807-457a-ae35-c024c2d7f73a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.918710174Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.91913718Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3be44b5b-5807-457a-ae35-c024c2d7f73a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.929023029Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"82dab271-aa92-4346-8620-f13006c5ebad","remote_addr":"[fd00:1122:3344:101::c]:42630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185985,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.929341314Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"83594b6d-951f-46a1-acf5-baa4ef92a9f4","remote_addr":"[fd00:1122:3344:101::c]:42630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.935217827Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.935230877Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 82a1832d-ab38-493d-9192-b919e0bcc5e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.954474078Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.954816813Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_82a1832d-ab38-493d-9192-b919e0bcc5e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.970409532Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.970464823Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.987231149Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"e1fa34c1-d7d4-47a6-bbf7-ab9b2b5751ad","remote_addr":"[fd00:1122:3344:101::b]:35763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":323815,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: cf86d888-5ca0-4e60-a3d8-21b19721ef99 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.994679645Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:11.994695395Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_cf86d888-5ca0-4e60-a3d8-21b19721ef99"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fad2b250-7ced-4cac-87db-22998050be7c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.020139373Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.020197914Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_fad2b250-7ced-4cac-87db-22998050be7c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f4bc5b75-b581-474b-87db-711856d8624e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.04476593Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.044818371Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f4bc5b75-b581-474b-87db-711856d8624e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: db8007cb-fccc-44e3-84e8-02e7abe2a72a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.070424172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.070481213Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b39324e6-d2a2-455c-88e7-60bce779fce1 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.096656011Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.096715902Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_b39324e6-d2a2-455c-88e7-60bce779fce1"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 6071668d-3100-464e-9aa6-4c653d46e04f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.121598933Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.121658384Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_6071668d-3100-464e-9aa6-4c653d46e04f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:12.135397888Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"ec714afb-3eda-4469-bb1f-05eb073ce623","remote_addr":"[fd00:1122:3344:101::c]:41282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280325,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.491736145Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.493985376Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.494666396Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.509488775Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.53187475Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.53256331Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.533236999Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.533876418Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.534371445Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.535012534Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.535474871Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.536999032Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.537608711Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.538125938Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.538746387Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.53963044Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.540136937Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:20.540637674Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:28.650063931Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:28.650210583Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"16b3c397-2ccf-4c15-bf18-457f92ad9b85","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39877","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.493462858Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.49568937Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.49638947Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.512463356Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.534583538Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.535334558Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.535984208Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.536605116Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.537117933Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.537788913Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.5382808Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.538773217Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.539238773Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.53972793Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.540349579Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.541254552Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.541749339Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:30.542227105Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.473546547Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.473656509Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"145fc296-10b9-4aa1-b0bb-29823f0bc6f0","remote_addr":"[fd00:1122:3344:101::a]:52639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.492378132Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.492402813Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e57f143f-095d-4d35-8534-7d5531f018ee","remote_addr":"[fd00:1122:3344:101::a]:52639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.495134811Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.49858731Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.499479502Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.503156424Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"96a91496-943e-4e40-a4c6-ddff98fcd215","remote_addr":"[fd00:1122:3344:101::a]:52639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.514518984Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.537331066Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.538152457Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.539255063Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.540399859Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.541296532Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.542039912Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.542546879Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.543061176Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.543648975Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.544178292Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.544830701Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.545764115Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.546278872Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:40.546773699Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.622931112Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35870"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.623099264Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"33c4af01-28ed-41e0-a7ec-89ee6d02f89e","remote_addr":"[fd00:1122:3344:101::a]:35870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.705514186Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55165"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.705596947Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.749960992Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.81297316Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cec94dfd-786a-42eb-84bf-d79c04deccfd","remote_addr":"[fd00:1122:3344:101::a]:55165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107413,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:40.813427877Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"41358e9f-af0b-4e5d-acc1-e6ddd76c4741","remote_addr":"[fd00:1122:3344:101::a]:55165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.305881396Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33891"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.30614282Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"439f94f0-696c-4fbf-9b02-7b71c8767cd9","remote_addr":"[fd00:1122:3344:101::b]:33891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.333194801Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.333235901Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"987fc9d5-070a-4c36-9349-328c429e73a5","remote_addr":"[fd00:1122:3344:101::b]:33891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.342097256Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.342323639Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f70eba1a-c937-46a7-a47d-be69d0e6aa29","remote_addr":"[fd00:1122:3344:101::b]:35105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.346019631Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"98fdefb2-1a38-42d2-bfea-95b055d124ed","remote_addr":"[fd00:1122:3344:101::b]:33891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.40413703Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63787"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.404231222Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.452854457Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.51837016Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"31cef3b4-8866-4a89-8b06-f1c4a6cbe4d0","remote_addr":"[fd00:1122:3344:101::b]:63787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.518537353Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.518750256Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"41ae350a-f575-455a-8cad-bec551ebfffd","remote_addr":"[fd00:1122:3344:101::c]:34704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.518897688Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1d629339-30fd-477d-884d-dfe3ae639ad2","remote_addr":"[fd00:1122:3344:101::b]:63787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.533534314Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.533553974Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d9af176b-4584-4f93-b8f9-75b18dc68642","remote_addr":"[fd00:1122:3344:101::c]:34704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.552791555Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41182"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.552908157Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6905ef40-645d-4bc9-90d5-2626c76f85de","remote_addr":"[fd00:1122:3344:101::c]:41182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.560623416Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"41b7b38b-95ba-4971-9317-4c594fa931bc","remote_addr":"[fd00:1122:3344:101::c]:34704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.631531925Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49736"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.631600006Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.672200898Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.730730943Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0a896c03-da38-450f-a895-44d2a78f69ec","remote_addr":"[fd00:1122:3344:101::c]:49736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99154,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:41.731024407Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"136c9e5c-1aba-4bd0-948b-0bc269fa5c0a","remote_addr":"[fd00:1122:3344:101::c]:49736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.496711533Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.500978273Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.501671063Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.515481317Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.538976208Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.539890841Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.540606111Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.542527688Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.543194778Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.543911048Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.544430335Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.544963623Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.54549724Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.546051388Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.546728407Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.547673891Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.548202008Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:50.548697285Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.734387642Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.734654635Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"49ae04e0-7062-4a29-b10c-a6058321846e","remote_addr":"[fd00:1122:3344:101::a]:53179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.735640999Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(8002347) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.736663144Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"PUT","req_id":"1e8d92e9-ba02-44a6-bf9d-997b3a698b2b","remote_addr":"[fd00:1122:3344:101::a]:53179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1835,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.807101176Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42756"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.823956384Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 6a5dc44f-15fe-425c-8fd5-53cfbe8f53d6, project_id: 7bc3193e-0379-4a52-8db0-fc186320043f }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-21T01:27:55.804262126Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(8002347))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6b1c3c96c16\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 519f9771-c587-458b-8d5f-ae2d2a2d42ef, read_only_parent: Some(Volume { block_size: 512, id: 896ab45c-09a2-4a83-bda1-906f3bb33170, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: afac1f5a-153f-4ac1-8bfc-096581a0f50e, key: Some(\"XCV4JFEpiL2ptAQ8cRzAQ1F5LQlPIEsblhbIK4ia7zk=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 519f9771-c587-458b-8d5f-ae2d2a2d42ef, key: Some(\"qb59Mtkfi6pfijsV5WVs37WHVhzfYT5a+3Nwu5ZTBFY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-6b1c3c96c16\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.824004794Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.825911251Z","hostname":"sock","pid":1566,"propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(8002347))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e167e7c-11ac-4d18-8839-a13ad5adaaaf, kind: Instance { id: 599adf06-e1ce-43d6-a3ad-60119136e26c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8002347), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6b1c3c96c16\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 519f9771-c587-458b-8d5f-ae2d2a2d42ef, read_only_parent: Some(Volume { block_size: 512, id: 896ab45c-09a2-4a83-bda1-906f3bb33170, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: afac1f5a-153f-4ac1-8bfc-096581a0f50e, key: Some(\"XCV4JFEpiL2ptAQ8cRzAQ1F5LQlPIEsblhbIK4ia7zk=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 519f9771-c587-458b-8d5f-ae2d2a2d42ef, key: Some(\"qb59Mtkfi6pfijsV5WVs37WHVhzfYT5a+3Nwu5ZTBFY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-6b1c3c96c16\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-21T01:27:55.804262126Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.828545158Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107","method":"PUT","req_id":"177fbae9-0b3d-4913-8393-bb55b974a688","remote_addr":"[fd00:1122:3344:101::a]:42756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21392,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.906456326Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.906952193Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107/state","method":"PUT","req_id":"6a3abe53-12c6-4a4a-b203-4a8a65ef4f1d","remote_addr":"[fd00:1122:3344:101::a]:50068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":399,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.90958025Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 8002347 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.909615561Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.952485985Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.97699528Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.993745006Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:55.993910689Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"57b2b13e-fdda-4921-8780-1196e73683ff","remote_addr":"[fd00:1122:3344:101::a]:64130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:56.011902452Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {0e167e7c-11ac-4d18-8839-a13ad5adaaaf: {10.151.2.175: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}, 10.151.2.176: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}}} }"} {"msg":"Profile for oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:57.583119694Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:57.583419418Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 74.53543ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:57.849570909Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 66.358267ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:57.940370278Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 157.139784ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:58.029061898Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 426.977028ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:58.203320984Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 763.729254ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:58.646737522Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:58.698139887Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:27:58.69834348Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6b1cf118-5ae6-45cd-9f95-0e1ba444d01d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38151","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 1.145403905s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:27:59.424484922Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.498992084Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.503359586Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.504044835Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.516823025Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.541142018Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.541818537Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.542494717Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.544232402Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.544788259Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.545439528Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.545928766Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.546450563Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.54692514Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.547428316Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.548049695Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.548943938Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.549447815Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.549923932Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 1.178510347s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:00.583489425Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 1.074350642s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:01.773340812Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:02.878461345Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 53.63886ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:02.891056403Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 134.146359ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:02.957095053Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\") failed: Property not found. retry in 209.337907ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:03.102212629Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.323461216Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.359506574Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.500017474Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-11-21T01:28:03.502224265Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.502258366Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.502377128Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1212"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.502412248Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-21T01:28:03.502224265Z }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.502420628Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-21T01:28:03.502224265Z }, migration_in: None, migration_out: None }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.502466619Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"85651fad-59c2-4b68-aa98-ed2bee2fc4d8","remote_addr":"[fd00:1122:3344:101::a]:64130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7490618,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-11-21T01:28:03.541834644Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.541864274Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.541888134Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-21T01:28:03.541834644Z }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.541895805Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-21T01:28:03.541834644Z }, migration_in: None, migration_out: None }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:03.695078913Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e93bbc25-aaf1-44c3-9d38-74150f94255c","remote_addr":"[fd00:1122:3344:101::a]:64130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":377,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.501476319Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.505070699Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.50584113Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.51859608Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.543161696Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.543854256Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.544526065Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.546223969Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.546774017Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.547441956Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.547937643Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.548470111Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.548957768Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.549466085Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.550102574Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.55123125Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.551724777Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:10.552227894Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.704271126Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34088"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.705705646Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107/state","method":"GET","req_id":"b368016d-3937-4ced-8c2b-3d96158c4478","remote_addr":"[fd00:1122:3344:101::a]:34088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1357,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.729682015Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43395"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.729897567Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ee85e1bb-2161-4fb7-b405-16d310d7db66","remote_addr":"[fd00:1122:3344:101::a]:43395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.789612439Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {0e167e7c-11ac-4d18-8839-a13ad5adaaaf: {10.151.2.175: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}, 10.151.2.176: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.78964578Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"51865983-26ef-4097-a04a-dd7acd29b29a","remote_addr":"[fd00:1122:3344:101::a]:43395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.809289476Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ea36b10e-1533-4f87-9a3e-20618f50cab4","remote_addr":"[fd00:1122:3344:101::a]:43395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.881819778Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.881982351Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"fd3c752d-3d19-4d69-8e67-b7f247421c15","remote_addr":"[fd00:1122:3344:101::a]:59714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.885410209Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65192"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.88550016Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.929940398Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.990222296Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b543a08a-f27f-482d-820e-258fc61a512c","remote_addr":"[fd00:1122:3344:101::a]:65192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104755,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:10.990554801Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5a7f1dfd-d313-419b-9e58-b309c43268ac","remote_addr":"[fd00:1122:3344:101::a]:65192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.308242948Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.31480059Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107/state","method":"GET","req_id":"b842a1d7-3aa9-4f3b-8a77-ab4c5fb6a009","remote_addr":"[fd00:1122:3344:101::b]:62578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.600286723Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.600467495Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5b2f42d9-6be5-4166-b122-9dbe7632c654","remote_addr":"[fd00:1122:3344:101::b]:57937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.612778169Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57860"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.61284351Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5c28a583-ed32-4d05-abb3-ad5f7394345f","remote_addr":"[fd00:1122:3344:101::b]:57860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.617429945Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.619773107Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107/state","method":"GET","req_id":"616c19b4-bd8f-4317-b13e-9f7ef182d0ba","remote_addr":"[fd00:1122:3344:101::c]:64700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.637348485Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {0e167e7c-11ac-4d18-8839-a13ad5adaaaf: {10.151.2.175: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}, 10.151.2.176: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.637380306Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"85389eef-7b77-4443-b620-1ad18a4984bd","remote_addr":"[fd00:1122:3344:101::b]:57860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.655469101Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"48b1c785-fcb6-493d-b0e9-4d7a683ecb6e","remote_addr":"[fd00:1122:3344:101::b]:57860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.698081511Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65372"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.698163472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.737775561Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.81293983Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"af82a316-eb8f-43a3-b45b-7d823de7148f","remote_addr":"[fd00:1122:3344:101::b]:65372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114779,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.813222764Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.813530578Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cd930e8b-4ac7-4515-9ad3-5dc5ba2b581a","remote_addr":"[fd00:1122:3344:101::b]:65372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.8136562Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"642d0b51-dee4-45c4-8d79-9bf64567f1d6","remote_addr":"[fd00:1122:3344:101::c]:36544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":321,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.944385872Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.944482793Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c3c53793-cba9-4b54-ac16-6f96b8365fde","remote_addr":"[fd00:1122:3344:101::c]:44769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.988347112Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {0e167e7c-11ac-4d18-8839-a13ad5adaaaf: {10.151.2.175: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}, 10.151.2.176: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:11.988375162Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e42d20ef-7f04-47e6-a103-6821060473d4","remote_addr":"[fd00:1122:3344:101::c]:44769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.004006144Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4266bbb5-8787-4049-918a-25aae3d100e8","remote_addr":"[fd00:1122:3344:101::c]:44769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.052904201Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63804"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.053048223Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.098346202Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.159540054Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1c0d16a7-6a30-462c-a153-01d8b145fc9b","remote_addr":"[fd00:1122:3344:101::c]:63804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106549,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:12.159818878Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"33fc42c2-a0d8-4705-a137-d207432a5e72","remote_addr":"[fd00:1122:3344:101::c]:63804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.559468392Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.56006529Z","hostname":"sock","pid":1566,"uri":"/vmms/4929648b-7103-40a5-8a25-6d3cccfcb107/state","method":"PUT","req_id":"d6efdf3d-d09d-49f6-b593-5484f58771d0","remote_addr":"[fd00:1122:3344:101::a]:49472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":513,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-11-21T01:28:13.560200492Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.560225932Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.560243703Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-21T01:28:13.560200492Z }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.560258263Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-21T01:28:13.560200492Z }, migration_in: None, migration_out: None }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-11-21T01:28:13.607588500Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.60761254Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.607646231Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-11-21T01:28:13.607588500Z }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.607654891Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.607679271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:13.607731232Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"17202663328678234514"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:13.607796583Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"13146196079580258816"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.607932584Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:13.607982575Z","hostname":"sock","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/0e00ae01-5cfa-4391-85ab-30571505fc6a/debug/bundle/zone\", \"/pool/int/67f3c571-2bea-4976-8ebe-c962b5aa082f/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/db6f4d38-f0e6-466e-b782-b9022f769535/crypt/debug\", \"/pool/ext/3be44b5b-5807-457a-ae35-c024c2d7f73a/crypt/debug\", \"/pool/ext/82a1832d-ab38-493d-9192-b919e0bcc5e4/crypt/debug\", \"/pool/ext/cf86d888-5ca0-4e60-a3d8-21b19721ef99/crypt/debug\", \"/pool/ext/fad2b250-7ced-4cac-87db-22998050be7c/crypt/debug\", \"/pool/ext/f4bc5b75-b581-474b-87db-711856d8624e/crypt/debug\", \"/pool/ext/db8007cb-fccc-44e3-84e8-02e7abe2a72a/crypt/debug\", \"/pool/ext/b39324e6-d2a2-455c-88e7-60bce779fce1/crypt/debug\", \"/pool/ext/6071668d-3100-464e-9aa6-4c653d46e04f/crypt/debug\"] }","zone_name":"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:14.253909907Z","hostname":"sock","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107\", bundle_id: 48f9b33d-fd38-4463-9107-b2aa9be62dde }, time_created: 2024-11-21T01:28:13.608916058Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_4929648b-7103-40a5-8a25-6d3cccfcb107","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:14.253992299Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.004881162Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.02387489Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(8002347), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.023936001Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(8002347), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.024288226Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-21T01:28:16.024254125Z }, migration_in: None, migration_out: None }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.096513773Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-21T01:28:16.024254125Z }, migration_in: None, migration_out: None }","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.098829876Z","hostname":"sock","pid":1566,"instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","propolis_id":"4929648b-7103-40a5-8a25-6d3cccfcb107","instance_id":"599adf06-e1ce-43d6-a3ad-60119136e26c","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.685071537Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.685225149Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a8b1c665-f305-4022-aa13-e89c28d8d2d6","remote_addr":"[fd00:1122:3344:101::c]:47172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.734014066Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {0e167e7c-11ac-4d18-8839-a13ad5adaaaf: {10.151.2.175: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}, 10.151.2.176: {5bbbba5e-242a-4f60-896b-b52cb3be20ad}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.734047928Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b1896c8a-43cb-4bad-a35c-220bd54ec3f0","remote_addr":"[fd00:1122:3344:101::c]:47172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.743189466Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5feb5f46-f86b-4a26-8e49-04bea25d96e8","remote_addr":"[fd00:1122:3344:101::c]:47172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.851629474Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:16.851816967Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"2f733f00-1032-4d53-b5c7-170e5ebe142a","remote_addr":"[fd00:1122:3344:101::c]:57664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.302892116Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.303131159Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"ea69d110-0369-4d0b-96f8-5ec25eb55411","remote_addr":"[fd00:1122:3344:101::a]:43559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.343562329Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.34367202Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"4a8f626f-edd8-48f8-8a26-c9144e1d3f67","remote_addr":"[fd00:1122:3344:101::a]:64445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.383588503Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.383664924Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.383671484Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"05bc65d3-9a3f-454f-8dde-47e07042e5a0","remote_addr":"[fd00:1122:3344:101::a]:64445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:18.391306151Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"89cef6c6-d51b-48cb-a2f0-8aaa24f464f0","remote_addr":"[fd00:1122:3344:101::a]:64445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:19.00293171Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:19.003038692Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1de38f07-ed3d-4faa-9e31-52672accb841","remote_addr":"[fd00:1122:3344:101::a]:57824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:19.051442984Z","hostname":"sock","pid":1566,"sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:19.051473375Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5a979bc1-ac52-45a6-ba0e-43f93209a4e0","remote_addr":"[fd00:1122:3344:101::a]:57824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T01:28:19.059883293Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"db838c81-4603-491b-ab1a-40a35da2b100","remote_addr":"[fd00:1122:3344:101::a]:57824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.503514267Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.506826373Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.507520493Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.520254282Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.545485378Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.546177868Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.546856637Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.547509486Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.549023528Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.549673107Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.550163544Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.550670521Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.551156368Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.551656425Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.552316214Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.553214117Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.553693063Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T01:28:20.55419173Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"623db7ea-cdeb-43ac-a5e8-f5c9bffff048","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6}