[ Jan 9 00:59:24 Enabled. ] [ Jan 9 00:59:24 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jan 9 00:59:24 Method "start" exited with status 0. ] [ Jan 9 00:59:24 Rereading configuration. ] [ Jan 9 00:59:24 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.10322173Z","hostname":"buskin","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.103359233Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.155704834Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.155746375Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.271392004Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.279996689Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.287666572Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:24.891538416Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.031700618Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.031733929Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.03179747Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.031855451Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.031874362Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.031888622Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034746617Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:25.034754937Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034765027Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034771127Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034803318Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034823648Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034931001Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034955401Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.034965991Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.04644815Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.046475941Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.145823723Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.504994896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505026977Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505032407Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505042437Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505053478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505065938Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.505070338Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.514068281Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.514088892Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"3d778ce3-4cf1-4708-abb3-05822320be17 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.604545212Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97010677Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970142421Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970175982Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970183002Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970193602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970201852Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970247373Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970256943Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970269894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970274234Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970278914Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970288454Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970291574Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970294554Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970308994Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970312734Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970317715Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970321305Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970324285Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970331505Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970345755Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970360406Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970366876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970370606Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970375736Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970379666Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970398907Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970406227Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970411137Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970416797Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970425147Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970438267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970445817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970452418Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970483609Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970488769Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970492599Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970500389Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970504969Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970510339Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97053985Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97054539Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97054925Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97055446Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.97055908Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970573971Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970606111Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970611491Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970615641Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970619841Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970654992Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970660162Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970664713Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970671793Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970676203Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970687003Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970691213Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970722854Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970727024Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970731334Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970735224Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970739194Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970745704Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970752764Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970760455Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970766545Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970771335Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970798676Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.970812726Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.981843145Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:748"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.981890636Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999920798s","next_cleanup":"Instant { tv_sec: 1409, tv_nsec: 819741162 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.981919667Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.981936317Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.981991428Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.982002568Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.98475332Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.986456909Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.986916609Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1204"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.986922209Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.986973681Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.986999631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987027032Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987052302Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987062763Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987070123Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987076583Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1253"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987082673Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987262707Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987394Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1285"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.98740354Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.98740936Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987440921Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987496972Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987505873Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987695557Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.98783584Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.987903731Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a (sled), rack_id: 122d04f2-96f1-44ac-9122-b16b480179a5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.988952425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4270"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.989008046Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4664"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.993096299Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.998428229Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:25.998609273Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.003628646Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.003700258Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.0038093Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:39207","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a (sled), rack_id: 122d04f2-96f1-44ac-9122-b16b480179a5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.003958224Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.003972064Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004016495Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004032905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004038566Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004045576Z","hostname":"buskin","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004061486Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004067486Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004075586Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004080007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004085787Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004090727Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004107337Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.004111267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.021591511Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.041602173Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.058277909Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.064116721Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.064149832Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.064170802Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:424"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.064187772Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 119.729043ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.067253242Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"263e2375-5c37-4c86-8402-9225586967c7\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"263e2375-5c37-4c86-8402-9225586967c7\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_bf8eca8a-81d6-48ff-9487-7cf623d74763/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.07782882Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_bf8eca8a-81d6-48ff-9487-7cf623d74763/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.100061872Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_bf8eca8a-81d6-48ff-9487-7cf623d74763/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.100099823Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.105216758Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:439"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.105226518Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_bf8eca8a-81d6-48ff-9487-7cf623d74763/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.105232679Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.171965824Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.188067078Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 102.361971ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.188673311Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ad912b90-f792-48b4-a98b-6d1e12ae5c63\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ad912b90-f792-48b4-a98b-6d1e12ae5c63\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.291779758Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 81.578621ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.292254878Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8598444a-a0b2-4b67-840c-c1da84cbd299\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8598444a-a0b2-4b67-840c-c1da84cbd299\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.375506736Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 186.537026ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.375923236Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a97090f-bf41-46dc-9565-8799ea6d4318\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a97090f-bf41-46dc-9565-8799ea6d4318\" }"} {"msg":"Mounting oxi_bf8eca8a-81d6-48ff-9487-7cf623d74763/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.390923355Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.403937778Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.40401793Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.40404456Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.404063861Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.563192971Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 300.823766ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.56404565Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c9abd51-6083-4603-a4be-d5bc4e4840e7\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c9abd51-6083-4603-a4be-d5bc4e4840e7\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.866106576Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 147.077172ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:26.86673876Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f868b895-b070-418a-aea3-7c289bcb4cc5\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f868b895-b070-418a-aea3-7c289bcb4cc5\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.014820361Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 299.554159ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.015332462Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c810be0-75cc-43a9-975d-9b27a93957e8\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 00:59:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c810be0-75cc-43a9-975d-9b27a93957e8\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.13666589Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.142118603Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.142159234Z","hostname":"buskin","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.142229175Z","hostname":"buskin","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:40239","producer_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:27.143352491Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"351.298835ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.144424125Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.144434025Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.144502007Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:932","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.144514237Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:27.316515098Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:27.496845056Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"322.759179ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:27.820337025Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"501.023143ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:28.323062627Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.162880866s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856179657Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2694"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856205687Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2939"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856210367Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2564"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856216267Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2611"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856252098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.856258588Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.858259344Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2864"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.859830979Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2605"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.859952352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:29.860132186Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 39.048631ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:30.094742809Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 105.838786ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:30.305640398Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 100.140986ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:30.486626671Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:30.487448859Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.366693848s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 293.625227ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:30.662293374Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 751.958983ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:31.009713053Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.40293792s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:31.772151745Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.254410655s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:33.395695135Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:33.855905518Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.227389353s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.100778894s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:34.889375225Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:36.427762644Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.45773792Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.45818193Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.459708765Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.459751395Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.459766106Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.459779786Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.459988641Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.460187185Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.460235737Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.460246867Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.460280207Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:39207","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.670263495Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.670325566Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:36.67579173Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4294","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:37.176692361Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13234406863265375397"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:37.177236923Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1740333662312298503"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.649036848Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55153"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.649444427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.649451268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.64954218Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.64956557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.649773705Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"139be181-b5b0-426e-b011-4d2f0db74c54","remote_addr":"[fd00:1122:3344:101::1]:55153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":487,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:37.650614404Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.682946633Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.683123437Z","hostname":"buskin","pid":1566,"uri":"/sled-role","method":"GET","req_id":"b1f6ee9d-b89b-44dd-9bb3-fb8113b985d4","remote_addr":"[fd00:1122:3344:101::1]:60080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5df58b73-f1c5-48f0-b9cb-6aa80ff5a274 (physical_disk), pool_id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 67a4a0a9-319a-45c5-a262-53742502c03e (physical_disk), pool_id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 2e1e343a-e268-4626-a0eb-d9e58e980bba (physical_disk), pool_id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: d25916fe-d25b-46a1-b19a-a8cb6a783057 (physical_disk), pool_id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 91c352af-9692-47b1-8bc4-e2decdb5818b (physical_disk), pool_id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 5ddedfab-cf77-46e3-8666-3e6a1ac8cf74 (physical_disk), pool_id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b69f8c0e-a48c-4c50-8efe-96144aa7e570 (physical_disk), pool_id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 2b5e9945-3c86-4ebf-bfa6-38910845f12b (physical_disk), pool_id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c45c51a1-8f9d-4bb6-b038-7510b70100cb (physical_disk), pool_id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714331971Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714505255Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33017"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714860223Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5df58b73-f1c5-48f0-b9cb-6aa80ff5a274 (physical_disk), pool_id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 67a4a0a9-319a-45c5-a262-53742502c03e (physical_disk), pool_id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 2e1e343a-e268-4626-a0eb-d9e58e980bba (physical_disk), pool_id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: d25916fe-d25b-46a1-b19a-a8cb6a783057 (physical_disk), pool_id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 91c352af-9692-47b1-8bc4-e2decdb5818b (physical_disk), pool_id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 5ddedfab-cf77-46e3-8666-3e6a1ac8cf74 (physical_disk), pool_id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b69f8c0e-a48c-4c50-8efe-96144aa7e570 (physical_disk), pool_id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 2b5e9945-3c86-4ebf-bfa6-38910845f12b (physical_disk), pool_id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c45c51a1-8f9d-4bb6-b038-7510b70100cb (physical_disk), pool_id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714940725Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714964006Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714978406Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.714982986Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.715028817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.715038097Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.715043257Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.715048708Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.874829333Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.882048816Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.882056296Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.978418959Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987014893Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987062334Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987070975Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987087795Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987096825Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987200168Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:37.987206518Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.020295004Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.028859257Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.174955884Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.182802291Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.182836771Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.182842352Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.182847452Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.182854402Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.192417228Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.192452768Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.290253095Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298256336Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298314917Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298330877Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298340417Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298348758Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298488851Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.298496121Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.333699865Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.342055674Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.480838085Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.489002059Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.48903361Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.48904113Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.48904572Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.48905169Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.49878389Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.49879832Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.594376677Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601786904Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601810055Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601815064Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601831985Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601838475Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.601988688Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.602010339Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.636363364Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.643897724Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:38.652320974Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.780958657Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.788923776Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.788958977Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.788965487Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.788970977Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.788976207Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.798018721Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.798044542Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.893948706Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.901881015Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.901930866Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.901938216Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.901949536Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.901957547Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.902052578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.902057839Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.938491721Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:38.946398999Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.085882216Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.093516878Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.093546239Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.093552109Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.093571279Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.0935754Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.103488823Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.103504584Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.198790024Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.206845685Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.206866196Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.206872176Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.206882406Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.206889146Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.207004959Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.207010669Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.242896529Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.251507983Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.389270291Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.397118788Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.397146739Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.397152649Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.397157899Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.397161809Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.406400998Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.406420908Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.50269817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.510507526Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.510527517Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.510549347Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.510557098Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.510562748Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.51067195Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.51067906Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.54433681Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.551698696Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:39.654200579Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_a132b3b6-342d-41c2-a6be-600f78f98f46","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.693874774Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.701439814Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.701467265Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.701473155Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.701477505Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.701489695Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.710384776Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.710400027Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.804420178Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.811872346Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.811892686Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.811897926Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.811903547Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.811942087Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.81204435Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.81205295Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.849522915Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.857468585Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:39.998027315Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.006273181Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.006300062Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.006306002Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.006310362Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.006315372Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.015891808Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.015907488Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.132624682Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.140829957Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.140851578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.140856578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.140875368Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.140893559Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.141018151Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.141025942Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:40.146765511Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.443330488s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.180259196Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.188310348Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.329022573Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.337145886Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.337183267Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.337190477Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.337199068Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.337216838Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.348179335Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.348213846Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.445419039Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453295937Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453368089Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453377599Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453407189Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.45341963Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453530372Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.453538522Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.49021031Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.499023428Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.640315796Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.64846802Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648504011Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648551682Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648639034Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bf8eca8a-81d6-48ff-9487-7cf623d74763 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3d778ce3-4cf1-4708-abb3-05822320be17 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648684195Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648690155Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.648863869Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649063224Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:897"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649078724Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649085584Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649089064Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:928"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649092245Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:933"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649114705Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649120265Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:939"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649146696Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649152276Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.649221447Z","hostname":"buskin","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"5a35e192-b2f1-4ccf-b025-52b1d9cfdbcf","remote_addr":"[fd00:1122:3344:101::1]:33017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2934574,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:40.655843357Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {014da691-510e-4602-b5f1-f42917d2a4e7 (dataset): DatasetConfig { id: 014da691-510e-4602-b5f1-f42917d2a4e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09caaf99-bbf8-4b23-bf22-82caa23f791e (dataset): DatasetConfig { id: 09caaf99-bbf8-4b23-bf22-82caa23f791e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0aaf2bc9-9597-4e92-a39e-1ede1b621123 (dataset): DatasetConfig { id: 0aaf2bc9-9597-4e92-a39e-1ede1b621123 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0b60f882-3590-4a95-9b2e-2a0d91f79b59 (dataset): DatasetConfig { id: 0b60f882-3590-4a95-9b2e-2a0d91f79b59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 136489a0-c8f5-47dc-9b4c-b8995b06724c (dataset): DatasetConfig { id: 136489a0-c8f5-47dc-9b4c-b8995b06724c (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 14d35e12-efdd-4588-8561-6a4f17527036 (dataset): DatasetConfig { id: 14d35e12-efdd-4588-8561-6a4f17527036 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 19386c7e-5273-49f1-b4a6-f1a2a555a049 (dataset): DatasetConfig { id: 19386c7e-5273-49f1-b4a6-f1a2a555a049 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba33af2-6811-4fcc-a03b-655e9e10443e (dataset): DatasetConfig { id: 1ba33af2-6811-4fcc-a03b-655e9e10443e (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1d689726-17db-4e9f-92a5-e6ff0d09e826 (dataset): DatasetConfig { id: 1d689726-17db-4e9f-92a5-e6ff0d09e826 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 22514807-4960-4833-9dc8-dcfd5d049e26 (dataset): DatasetConfig { id: 22514807-4960-4833-9dc8-dcfd5d049e26 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2542073f-fd27-4671-ad31-608a18eb9f99 (dataset): DatasetConfig { id: 2542073f-fd27-4671-ad31-608a18eb9f99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2543e1cd-9c77-4ddc-890e-32f354984002 (dataset): DatasetConfig { id: 2543e1cd-9c77-4ddc-890e-32f354984002 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28ad1367-931e-4d6a-8247-764cd662a4d9 (dataset): DatasetConfig { id: 28ad1367-931e-4d6a-8247-764cd662a4d9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 290460e9-8b0e-4e1e-bfe9-b821afb46d7b (dataset): DatasetConfig { id: 290460e9-8b0e-4e1e-bfe9-b821afb46d7b (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 31813aca-037d-494e-9672-f3d7cc5b6d20 (dataset): DatasetConfig { id: 31813aca-037d-494e-9672-f3d7cc5b6d20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 35fec43b-670c-4941-afe4-cb468158cf95 (dataset): DatasetConfig { id: 35fec43b-670c-4941-afe4-cb468158cf95 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 40b189a0-3ca6-4033-9ff1-c10547b2edc2 (dataset): DatasetConfig { id: 40b189a0-3ca6-4033-9ff1-c10547b2edc2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 46ecedbf-2d6b-4d50-95e2-6d4145089d5b (dataset): DatasetConfig { id: 46ecedbf-2d6b-4d50-95e2-6d4145089d5b (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ce8c0d9-87fb-47e3-a007-4130075af6cb (dataset): DatasetConfig { id: 4ce8c0d9-87fb-47e3-a007-4130075af6cb (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ea6b3ca-222e-4b16-903e-e922ff2d7f30 (dataset): DatasetConfig { id: 4ea6b3ca-222e-4b16-903e-e922ff2d7f30 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53deca0a-a5ec-4533-88a7-c367390e5a90 (dataset): DatasetConfig { id: 53deca0a-a5ec-4533-88a7-c367390e5a90 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58454861-105e-427d-afb2-abc92f11ef82 (dataset): DatasetConfig { id: 58454861-105e-427d-afb2-abc92f11ef82 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 59a823f1-88ff-46ba-a5d8-f2b1ddf1c853 (dataset): DatasetConfig { id: 59a823f1-88ff-46ba-a5d8-f2b1ddf1c853 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5a5d08cd-a0d5-4c58-a7db-8a5e1d4fd3af (dataset): DatasetConfig { id: 5a5d08cd-a0d5-4c58-a7db-8a5e1d4fd3af (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 62924c35-58ec-49de-a306-45513768b5f3 (dataset): DatasetConfig { id: 62924c35-58ec-49de-a306-45513768b5f3 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 684373e2-a532-45cd-b5c8-266077b5c8f6 (dataset): DatasetConfig { id: 684373e2-a532-45cd-b5c8-266077b5c8f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 696b2588-4b69-42ae-bf42-0b2b4436522f (dataset): DatasetConfig { id: 696b2588-4b69-42ae-bf42-0b2b4436522f (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e5c62af-366e-4962-acb6-93fbda991fd7 (dataset): DatasetConfig { id: 6e5c62af-366e-4962-acb6-93fbda991fd7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6ffdfb01-d596-4d00-98b0-996cf98c8bab (dataset): DatasetConfig { id: 6ffdfb01-d596-4d00-98b0-996cf98c8bab (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 788a6011-b317-478f-8a9b-56988574020a (dataset): DatasetConfig { id: 788a6011-b317-478f-8a9b-56988574020a (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d1c9ad1-4b6d-456d-8c86-9027362c5075 (dataset): DatasetConfig { id: 7d1c9ad1-4b6d-456d-8c86-9027362c5075 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8305226e-0096-4f31-b642-383ee509f2fc (dataset): DatasetConfig { id: 8305226e-0096-4f31-b642-383ee509f2fc (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 90b60049-06e5-41a4-a1d9-e7b8152b2704 (dataset): DatasetConfig { id: 90b60049-06e5-41a4-a1d9-e7b8152b2704 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 951478a6-4ddb-4867-a6c0-dc27ce82124a (dataset): DatasetConfig { id: 951478a6-4ddb-4867-a6c0-dc27ce82124a (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96d22ba8-4700-4771-803a-1fb49bc80627 (dataset): DatasetConfig { id: 96d22ba8-4700-4771-803a-1fb49bc80627 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9895c788-af7c-40ce-8a96-6d8448172370 (dataset): DatasetConfig { id: 9895c788-af7c-40ce-8a96-6d8448172370 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9dcd4c44-4048-4a21-81f9-cce348429a29 (dataset): DatasetConfig { id: 9dcd4c44-4048-4a21-81f9-cce348429a29 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9e2afa8e-f58d-428b-9b37-3ed0d734cd76 (dataset): DatasetConfig { id: 9e2afa8e-f58d-428b-9b37-3ed0d734cd76 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a2c040ab-a97b-45bd-a05f-41b324b52cc9 (dataset): DatasetConfig { id: a2c040ab-a97b-45bd-a05f-41b324b52cc9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4047fbe-73aa-4d2e-b103-8feb62721cdd (dataset): DatasetConfig { id: a4047fbe-73aa-4d2e-b103-8feb62721cdd (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a98425b8-5787-479f-a85e-e03ae7bb9401 (dataset): DatasetConfig { id: a98425b8-5787-479f-a85e-e03ae7bb9401 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ab701ed8-c432-4d89-a984-d24d85cf507d (dataset): DatasetConfig { id: ab701ed8-c432-4d89-a984-d24d85cf507d (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b1c54e98-918e-45d5-af96-3e01075969a7 (dataset): DatasetConfig { id: b1c54e98-918e-45d5-af96-3e01075969a7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b3ea7e66-c83b-4153-a39c-88e2b4c562e7 (dataset): DatasetConfig { id: b3ea7e66-c83b-4153-a39c-88e2b4c562e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b45036f5-05e9-4b4b-b63b-4b7829440c04 (dataset): DatasetConfig { id: b45036f5-05e9-4b4b-b63b-4b7829440c04 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b60f7d89-deb3-4fd9-8b7b-45527d21c921 (dataset): DatasetConfig { id: b60f7d89-deb3-4fd9-8b7b-45527d21c921 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b8bda1e3-5506-4a7d-94da-226a14300dee (dataset): DatasetConfig { id: b8bda1e3-5506-4a7d-94da-226a14300dee (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ba15bb3a-bb94-42ed-bae9-b4160c48047c (dataset): DatasetConfig { id: ba15bb3a-bb94-42ed-bae9-b4160c48047c (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf075714-dd19-41f2-91e4-94f839b789ad (dataset): DatasetConfig { id: bf075714-dd19-41f2-91e4-94f839b789ad (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf9fedc3-857c-41db-b7fd-2559b1170988 (dataset): DatasetConfig { id: bf9fedc3-857c-41db-b7fd-2559b1170988 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c2d5cc3b-8d54-442f-84d5-d3d4e43df0ab (dataset): DatasetConfig { id: c2d5cc3b-8d54-442f-84d5-d3d4e43df0ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8799b70-1a54-43a4-9aac-ff3d02e16888 (dataset): DatasetConfig { id: c8799b70-1a54-43a4-9aac-ff3d02e16888 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d35f9453-2298-4367-9177-bbe19568b0d2 (dataset): DatasetConfig { id: d35f9453-2298-4367-9177-bbe19568b0d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dd84d3ca-f578-4061-8241-17e865ec4713 (dataset): DatasetConfig { id: dd84d3ca-f578-4061-8241-17e865ec4713 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e203b230-1861-4a38-84ee-a2c9ed111015 (dataset): DatasetConfig { id: e203b230-1861-4a38-84ee-a2c9ed111015 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2e1ca54-e774-4e35-9450-ee37d6ee6d7f (dataset): DatasetConfig { id: e2e1ca54-e774-4e35-9450-ee37d6ee6d7f (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e6deb48d-ee64-4ad0-98bb-6b475e394c85 (dataset): DatasetConfig { id: e6deb48d-ee64-4ad0-98bb-6b475e394c85 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e790f0f7-fad3-4390-9878-97bf2c0dd160 (dataset): DatasetConfig { id: e790f0f7-fad3-4390-9878-97bf2c0dd160 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ecc8f836-e7c3-4708-b100-39d38af6f717 (dataset): DatasetConfig { id: ecc8f836-e7c3-4708-b100-39d38af6f717 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ed92b840-dd7f-4cc4-8406-cfe25dab5647 (dataset): DatasetConfig { id: ed92b840-dd7f-4cc4-8406-cfe25dab5647 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f0229e63-1a00-4e79-82fd-32275ee50e8a (dataset): DatasetConfig { id: f0229e63-1a00-4e79-82fd-32275ee50e8a (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f0ce39ff-79f6-4a0c-8805-1aa50cfbe6aa (dataset): DatasetConfig { id: f0ce39ff-79f6-4a0c-8805-1aa50cfbe6aa (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1675227-6e9c-495d-bd04-f6916ec6e4e2 (dataset): DatasetConfig { id: f1675227-6e9c-495d-bd04-f6916ec6e4e2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f246b7d3-d9a5-441a-a746-b8228bb833b0 (dataset): DatasetConfig { id: f246b7d3-d9a5-441a-a746-b8228bb833b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, feb11a14-bb86-40a8-920e-e83c81c6514f (dataset): DatasetConfig { id: feb11a14-bb86-40a8-920e-e83c81c6514f (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffffa44a-adf3-4f14-99c2-5a6ecc61d189 (dataset): DatasetConfig { id: ffffa44a-adf3-4f14-99c2-5a6ecc61d189 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.680661487Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.68080106Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59063"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.681384893Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {014da691-510e-4602-b5f1-f42917d2a4e7 (dataset): DatasetConfig { id: 014da691-510e-4602-b5f1-f42917d2a4e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09caaf99-bbf8-4b23-bf22-82caa23f791e (dataset): DatasetConfig { id: 09caaf99-bbf8-4b23-bf22-82caa23f791e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0aaf2bc9-9597-4e92-a39e-1ede1b621123 (dataset): DatasetConfig { id: 0aaf2bc9-9597-4e92-a39e-1ede1b621123 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0b60f882-3590-4a95-9b2e-2a0d91f79b59 (dataset): DatasetConfig { id: 0b60f882-3590-4a95-9b2e-2a0d91f79b59 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 136489a0-c8f5-47dc-9b4c-b8995b06724c (dataset): DatasetConfig { id: 136489a0-c8f5-47dc-9b4c-b8995b06724c (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 14d35e12-efdd-4588-8561-6a4f17527036 (dataset): DatasetConfig { id: 14d35e12-efdd-4588-8561-6a4f17527036 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 19386c7e-5273-49f1-b4a6-f1a2a555a049 (dataset): DatasetConfig { id: 19386c7e-5273-49f1-b4a6-f1a2a555a049 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ba33af2-6811-4fcc-a03b-655e9e10443e (dataset): DatasetConfig { id: 1ba33af2-6811-4fcc-a03b-655e9e10443e (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1d689726-17db-4e9f-92a5-e6ff0d09e826 (dataset): DatasetConfig { id: 1d689726-17db-4e9f-92a5-e6ff0d09e826 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 22514807-4960-4833-9dc8-dcfd5d049e26 (dataset): DatasetConfig { id: 22514807-4960-4833-9dc8-dcfd5d049e26 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2542073f-fd27-4671-ad31-608a18eb9f99 (dataset): DatasetConfig { id: 2542073f-fd27-4671-ad31-608a18eb9f99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2543e1cd-9c77-4ddc-890e-32f354984002 (dataset): DatasetConfig { id: 2543e1cd-9c77-4ddc-890e-32f354984002 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28ad1367-931e-4d6a-8247-764cd662a4d9 (dataset): DatasetConfig { id: 28ad1367-931e-4d6a-8247-764cd662a4d9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 290460e9-8b0e-4e1e-bfe9-b821afb46d7b (dataset): DatasetConfig { id: 290460e9-8b0e-4e1e-bfe9-b821afb46d7b (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 31813aca-037d-494e-9672-f3d7cc5b6d20 (dataset): DatasetConfig { id: 31813aca-037d-494e-9672-f3d7cc5b6d20 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 35fec43b-670c-4941-afe4-cb468158cf95 (dataset): DatasetConfig { id: 35fec43b-670c-4941-afe4-cb468158cf95 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 40b189a0-3ca6-4033-9ff1-c10547b2edc2 (dataset): DatasetConfig { id: 40b189a0-3ca6-4033-9ff1-c10547b2edc2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 46ecedbf-2d6b-4d50-95e2-6d4145089d5b (dataset): DatasetConfig { id: 46ecedbf-2d6b-4d50-95e2-6d4145089d5b (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ce8c0d9-87fb-47e3-a007-4130075af6cb (dataset): DatasetConfig { id: 4ce8c0d9-87fb-47e3-a007-4130075af6cb (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ea6b3ca-222e-4b16-903e-e922ff2d7f30 (dataset): DatasetConfig { id: 4ea6b3ca-222e-4b16-903e-e922ff2d7f30 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 53deca0a-a5ec-4533-88a7-c367390e5a90 (dataset): DatasetConfig { id: 53deca0a-a5ec-4533-88a7-c367390e5a90 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 58454861-105e-427d-afb2-abc92f11ef82 (dataset): DatasetConfig { id: 58454861-105e-427d-afb2-abc92f11ef82 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 59a823f1-88ff-46ba-a5d8-f2b1ddf1c853 (dataset): DatasetConfig { id: 59a823f1-88ff-46ba-a5d8-f2b1ddf1c853 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5a5d08cd-a0d5-4c58-a7db-8a5e1d4fd3af (dataset): DatasetConfig { id: 5a5d08cd-a0d5-4c58-a7db-8a5e1d4fd3af (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 62924c35-58ec-49de-a306-45513768b5f3 (dataset): DatasetConfig { id: 62924c35-58ec-49de-a306-45513768b5f3 (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 684373e2-a532-45cd-b5c8-266077b5c8f6 (dataset): DatasetConfig { id: 684373e2-a532-45cd-b5c8-266077b5c8f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 696b2588-4b69-42ae-bf42-0b2b4436522f (dataset): DatasetConfig { id: 696b2588-4b69-42ae-bf42-0b2b4436522f (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e5c62af-366e-4962-acb6-93fbda991fd7 (dataset): DatasetConfig { id: 6e5c62af-366e-4962-acb6-93fbda991fd7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6ffdfb01-d596-4d00-98b0-996cf98c8bab (dataset): DatasetConfig { id: 6ffdfb01-d596-4d00-98b0-996cf98c8bab (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 788a6011-b317-478f-8a9b-56988574020a (dataset): DatasetConfig { id: 788a6011-b317-478f-8a9b-56988574020a (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d1c9ad1-4b6d-456d-8c86-9027362c5075 (dataset): DatasetConfig { id: 7d1c9ad1-4b6d-456d-8c86-9027362c5075 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8305226e-0096-4f31-b642-383ee509f2fc (dataset): DatasetConfig { id: 8305226e-0096-4f31-b642-383ee509f2fc (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 90b60049-06e5-41a4-a1d9-e7b8152b2704 (dataset): DatasetConfig { id: 90b60049-06e5-41a4-a1d9-e7b8152b2704 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 951478a6-4ddb-4867-a6c0-dc27ce82124a (dataset): DatasetConfig { id: 951478a6-4ddb-4867-a6c0-dc27ce82124a (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96d22ba8-4700-4771-803a-1fb49bc80627 (dataset): DatasetConfig { id: 96d22ba8-4700-4771-803a-1fb49bc80627 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9895c788-af7c-40ce-8a96-6d8448172370 (dataset): DatasetConfig { id: 9895c788-af7c-40ce-8a96-6d8448172370 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9dcd4c44-4048-4a21-81f9-cce348429a29 (dataset): DatasetConfig { id: 9dcd4c44-4048-4a21-81f9-cce348429a29 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9e2afa8e-f58d-428b-9b37-3ed0d734cd76 (dataset): DatasetConfig { id: 9e2afa8e-f58d-428b-9b37-3ed0d734cd76 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a2c040ab-a97b-45bd-a05f-41b324b52cc9 (dataset): DatasetConfig { id: a2c040ab-a97b-45bd-a05f-41b324b52cc9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4047fbe-73aa-4d2e-b103-8feb62721cdd (dataset): DatasetConfig { id: a4047fbe-73aa-4d2e-b103-8feb62721cdd (dataset), name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a98425b8-5787-479f-a85e-e03ae7bb9401 (dataset): DatasetConfig { id: a98425b8-5787-479f-a85e-e03ae7bb9401 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ab701ed8-c432-4d89-a984-d24d85cf507d (dataset): DatasetConfig { id: ab701ed8-c432-4d89-a984-d24d85cf507d (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b1c54e98-918e-45d5-af96-3e01075969a7 (dataset): DatasetConfig { id: b1c54e98-918e-45d5-af96-3e01075969a7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b3ea7e66-c83b-4153-a39c-88e2b4c562e7 (dataset): DatasetConfig { id: b3ea7e66-c83b-4153-a39c-88e2b4c562e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b45036f5-05e9-4b4b-b63b-4b7829440c04 (dataset): DatasetConfig { id: b45036f5-05e9-4b4b-b63b-4b7829440c04 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b60f7d89-deb3-4fd9-8b7b-45527d21c921 (dataset): DatasetConfig { id: b60f7d89-deb3-4fd9-8b7b-45527d21c921 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b8bda1e3-5506-4a7d-94da-226a14300dee (dataset): DatasetConfig { id: b8bda1e3-5506-4a7d-94da-226a14300dee (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ba15bb3a-bb94-42ed-bae9-b4160c48047c (dataset): DatasetConfig { id: ba15bb3a-bb94-42ed-bae9-b4160c48047c (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf075714-dd19-41f2-91e4-94f839b789ad (dataset): DatasetConfig { id: bf075714-dd19-41f2-91e4-94f839b789ad (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf9fedc3-857c-41db-b7fd-2559b1170988 (dataset): DatasetConfig { id: bf9fedc3-857c-41db-b7fd-2559b1170988 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c2d5cc3b-8d54-442f-84d5-d3d4e43df0ab (dataset): DatasetConfig { id: c2d5cc3b-8d54-442f-84d5-d3d4e43df0ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8799b70-1a54-43a4-9aac-ff3d02e16888 (dataset): DatasetConfig { id: c8799b70-1a54-43a4-9aac-ff3d02e16888 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d35f9453-2298-4367-9177-bbe19568b0d2 (dataset): DatasetConfig { id: d35f9453-2298-4367-9177-bbe19568b0d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dd84d3ca-f578-4061-8241-17e865ec4713 (dataset): DatasetConfig { id: dd84d3ca-f578-4061-8241-17e865ec4713 (dataset), name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e203b230-1861-4a38-84ee-a2c9ed111015 (dataset): DatasetConfig { id: e203b230-1861-4a38-84ee-a2c9ed111015 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2e1ca54-e774-4e35-9450-ee37d6ee6d7f (dataset): DatasetConfig { id: e2e1ca54-e774-4e35-9450-ee37d6ee6d7f (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e6deb48d-ee64-4ad0-98bb-6b475e394c85 (dataset): DatasetConfig { id: e6deb48d-ee64-4ad0-98bb-6b475e394c85 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e790f0f7-fad3-4390-9878-97bf2c0dd160 (dataset): DatasetConfig { id: e790f0f7-fad3-4390-9878-97bf2c0dd160 (dataset), name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ecc8f836-e7c3-4708-b100-39d38af6f717 (dataset): DatasetConfig { id: ecc8f836-e7c3-4708-b100-39d38af6f717 (dataset), name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ed92b840-dd7f-4cc4-8406-cfe25dab5647 (dataset): DatasetConfig { id: ed92b840-dd7f-4cc4-8406-cfe25dab5647 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f0229e63-1a00-4e79-82fd-32275ee50e8a (dataset): DatasetConfig { id: f0229e63-1a00-4e79-82fd-32275ee50e8a (dataset), name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f0ce39ff-79f6-4a0c-8805-1aa50cfbe6aa (dataset): DatasetConfig { id: f0ce39ff-79f6-4a0c-8805-1aa50cfbe6aa (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1675227-6e9c-495d-bd04-f6916ec6e4e2 (dataset): DatasetConfig { id: f1675227-6e9c-495d-bd04-f6916ec6e4e2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f246b7d3-d9a5-441a-a746-b8228bb833b0 (dataset): DatasetConfig { id: f246b7d3-d9a5-441a-a746-b8228bb833b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, feb11a14-bb86-40a8-920e-e83c81c6514f (dataset): DatasetConfig { id: feb11a14-bb86-40a8-920e-e83c81c6514f (dataset), name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffffa44a-adf3-4f14-99c2-5a6ecc61d189 (dataset): DatasetConfig { id: ffffa44a-adf3-4f14-99c2-5a6ecc61d189 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.681617088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.681629959Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.681635379Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.681640029Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.697187259Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.69721981Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.697224951Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.756418496Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.756449307Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.756453737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.814729062Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.814759332Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.814768243Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/debug\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.844051483Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.844083104Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.844088234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.902463681Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.902492962Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.902497202Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.963063448Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.963093199Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:40.963097519Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.021988208Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.022020788Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.022025099Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.070542473Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.070571134Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.070575524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.097297937Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.097325547Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.097329507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/debug\"), full_name: \"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.124728476Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.124764357Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.124768797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.184232098Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.184263529Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.184267799Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/debug\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.214370888Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.214397839Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.214401809Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.263355873Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.263388104Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.263392814Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.323619733Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.323655754Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.323661124Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.379455673Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.379489524Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.379494634Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone/oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.437434901Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.437469092Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.437473682Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.499061871Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.499093972Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.499098202Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone\"), full_name: \"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.528730001Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.528757701Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.528761912Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.587861215Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.587893456Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.587897856Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"refreshed zone-network-setup service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.608006689Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.6080446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4401"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.638527608Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.638573649Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.638577979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:41.658437427Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.668216888Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.668233188Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.668237928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.728925058Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.728965558Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.728970769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.779474228Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.779511579Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.779524869Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.828929064Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.828962825Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.828966965Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.878614665Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.878651646Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.878656566Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.928445149Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.928479Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.92848336Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.976211447Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.976265078Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:41.976270668Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.038582634Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.038624035Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.038629165Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/debug\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.068732074Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.068769875Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.068775145Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/debug\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.098977097Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.099014157Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.099020018Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.128221706Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.128253317Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.128258087Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/debug\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.15673356Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.15676681Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.15677067Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.205263955Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.205297695Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.205301925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.255742263Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.255777884Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.255793004Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.317160179Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.31719708Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.31720137Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.376733513Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.376771004Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.376775714Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.436770058Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.436806609Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.436811019Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/debug\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.466403386Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.466436777Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.466440887Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.516285072Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.516321093Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.516326263Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.563884106Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.563921917Z","hostname":"buskin","pid":1566,"name":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.563927277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone\"), full_name: \"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.593071544Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone/oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.593110665Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone/oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.593127735Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone/oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\"), full_name: \"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone/oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.652498905Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.652536626Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.652541606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:42.662660654Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.714944134Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.714978575Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.714982925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.775318026Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.775351097Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.775361737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.833294024Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.833328755Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.833332855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/debug\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.86237304Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.862404761Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.862409601Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.911278684Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.911313514Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.911318675Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.959609314Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.959638655Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.959642905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.989049218Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.989081019Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:42.989085809Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.019469945Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.019503235Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.019508046Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.06848704Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.068519501Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.068524412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.097489155Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.097521336Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.097525936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/debug\"), full_name: \"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.126303125Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.126335656Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.126340936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.177087291Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.177122512Z","hostname":"buskin","pid":1566,"name":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.177127512Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone\"), full_name: \"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.206366662Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.206399172Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.206417072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.257436794Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.257478535Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.257484275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/zone/oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.31976481Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.319797731Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.319802351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.382127477Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.382160688Z","hostname":"buskin","pid":1566,"name":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.382164998Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.430910338Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.430944438Z","hostname":"buskin","pid":1566,"name":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.430949618Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\"), full_name: \"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/zone/oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.491672499Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.491708159Z","hostname":"buskin","pid":1566,"name":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.4917131Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\"), full_name: \"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/zone/oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.551240692Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.551276893Z","hostname":"buskin","pid":1566,"name":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.551281773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\"), full_name: \"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/zone/oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.611325498Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.611372679Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.611378679Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:43.665437799Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.666165176Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone/oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.666172566Z","hostname":"buskin","pid":1566,"name":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone/oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.666178336Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone/oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\"), full_name: \"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/zone/oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.727198242Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.727238683Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.727244074Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/zone/oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.78649319Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.786535761Z","hostname":"buskin","pid":1566,"name":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.786540331Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.836011177Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.836045758Z","hostname":"buskin","pid":1566,"name":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.836049858Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\"), full_name: \"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/zone/oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.89774116Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.898138109Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.898453706Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.898541658Z","hostname":"buskin","pid":1566,"uri":"/datasets","method":"PUT","req_id":"ca75061e-752e-4f2f-b063-84e4325ea4f3","remote_addr":"[fd00:1122:3344:101::1]:59063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3217623,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.93053792Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.930646763Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:47384"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.93097255Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.93097959Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.94027545Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.940300321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.948604688Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.948627358Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.956277901Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.956348463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.956357863Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:43.956427504Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.000024607Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.000052378Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.000057838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.048092792Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.051677962Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.055373406Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.066934307Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.071482289Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:44.075386877Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:44.667338443Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.680218025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.680299917Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.680465901Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.763946905Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4467"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.763988526Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4494"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.805262937Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4507"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.805300678Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4353"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.858655482Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4395"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.858696502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4474","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.89273214Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4485"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:45.892786161Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4527"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.038763264Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4559"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.038804915Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"} {"msg":"Profile for oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.167790135Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.218732015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4606"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.218786446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.262320648Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.29478013Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.294888453Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.295248281Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.295274381Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.331451098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380668268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.38074951Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.3807619Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.38077135Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380778321Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380786791Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380795361Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380802041Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380810522Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380817131Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380824172Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380830852Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380837482Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380842602Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380852352Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380857672Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380862352Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380866963Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380871683Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380876393Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380881133Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380885913Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.380890563Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:46.429234384Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1507574473011067416"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:46.429353307Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11433719922119022153"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.657508134Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.657604016Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.657661298Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 58.283691ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.666065947Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.666102478Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:46.669996986Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 148.281494ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.734979602Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 35.92936ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.798655369Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 64.843455ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.805377781Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 136.237133ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.842308274Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 51.523745ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:46.982586909Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 100.691341ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.011621474Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 105.898079ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.121161275Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 160.078756ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.166200001Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:47.18207834Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13234406863265375397"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:47.182154891Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1740333662312298503"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 462.261028ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.191639125Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 403.857831ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.215920503Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 496.555195ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.336557285Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 748.02408ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.628817909Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 634.910769ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.663178354Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 793.133171ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:47.842976341Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 1.349573329s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:48.308821381Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 608.889077ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:48.390513034Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 603.35634ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:48.646199453Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 1.24237572s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:49.008754073Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 682.557982ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:49.257799012Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 769.913968ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:49.666857651Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 1.334809308s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:49.949015747Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0\") failed: Property not found. retry in 1.451184261s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:50.260220278Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 597.029872ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:50.44692693Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b\") failed: Property not found. retry in 910.331108ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:51.053536376Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372\") failed: Property not found. retry in 972.840059ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:51.293812997Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.286670448Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_b7e5d812-3b66-48ae-9425-d561b75ab23b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.286721649Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_6135d2f0-37b8-443e-a741-5ec4e46a4ef0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.286726649Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_a98745ae-2f0a-4dbf-9d35-ec0663205372"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.286735209Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.287008655Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.287290622Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bb39b16c-0aa0-4c88-99d3-5949db8e888c","remote_addr":"[fd00:1122:3344:101::1]:47384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8356540,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.289338668Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 139.539793ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:52.319753054Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.482530607Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 57.843157ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:52.659278254Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config)"} {"msg":"failed to write DNS configuration (will retry in 118.444567ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:52.718542631Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.856950924Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.904874085Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.904884456Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.935741742Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.936511849Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.936517329Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.965627766Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.965742889Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55652"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.965943643Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.965950333Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.977180387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.977219588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.986794213Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.986823114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.994824205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.994871136Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.99507186Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:52.995165012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.007388168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.04027099Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.041021637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.041028347Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.071018584Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 162, 151]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.071028384Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.071509475Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.071514925Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.071842772Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.073113541Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.095633389Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.11340557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_772a58ca-731f-4866-9630-9630892160a5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.729772527Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_772a58ca-731f-4866-9630-9630892160a5 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:53.729956631Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 33.807664ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:53.857653692Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 117.637392ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:53.898540644Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 161.893048ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:54.036755042Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 289.177371ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:54.210548233Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 1.139269636s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:54.509411386Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 770.629298ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:55.658963902Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:56.435078223Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1507574473011067416"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:56.435233746Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11433719922119022153"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 689.470222ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:56.438783396Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5\") failed: Property not found. retry in 1.492786584s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:57.138071973Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:57.184089682Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13234406863265375397"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:57.184218115Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1740333662312298503"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.652299637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_ntp_772a58ca-731f-4866-9630-9630892160a5"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.652369408Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.652696966Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.653016783Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bee469ed-17cd-4142-b81e-53035c369c62","remote_addr":"[fd00:1122:3344:101::1]:55652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5687183,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.655739354Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.686218562Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.686377606Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:56540"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_772a58ca-731f-4866-9630-9630892160a5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T00:59:58.697411715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.697521627Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"b6f5673f-ec0b-4f47-8a59-7e850d89d06a","remote_addr":"[fd00:1122:3344:101::1]:56540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":11026,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T00:59:58.700447473Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.820247286Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T00:59:58.82040906Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49562"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:01.742103689Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10336192206346733671"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:01.990177216Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16910371851796927903"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:02.289626602Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"644178316599370927"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_772a58ca-731f-4866-9630-9630892160a5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:02.829558394Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:02.829716607Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"1618456a-f657-47e5-909e-1c62aee31096","remote_addr":"[fd00:1122:3344:101::1]:49562","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009228,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:02.832692955Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:02.990251729Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:02.990394372Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60937"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_772a58ca-731f-4866-9630-9630892160a5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:02.998818312Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:02.998916584Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"db4716c2-5fc2-4a1a-a0e4-d22e0c3b10e6","remote_addr":"[fd00:1122:3344:101::1]:60937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8342,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:03.001631985Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:03.167559909Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:03.167692462Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57696"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:07.032209227Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1507574473011067416"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:07.03232957Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11433719922119022153"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_772a58ca-731f-4866-9630-9630892160a5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:07.189172478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.189273321Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"00fd28c3-6f07-4534-9f14-5a3ed6e77189","remote_addr":"[fd00:1122:3344:101::1]:57696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008526,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.191259945Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:07.1994315Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13234406863265375397"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:00:07.199563593Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1740333662312298503"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.400791173Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.400945606Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:56223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.423794662Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"0d1b5ed5-24df-443e-8c21-02359090e4f1","remote_addr":"[fd00:1122:3344:101::1]:56223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":22761,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 395716850, ip_addr: 23.150.40.242, stratum: 3, ref_time: 1736384407.140443, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.427519126Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.427526696Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1392"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.457612995Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.457734078Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34275"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.458161017Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.458167857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.468832708Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.468870039Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.47781196Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.477840131Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.486078757Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.486111318Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.496140224Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.496182125Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.504452912Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.504480752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.512173686Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.512201116Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.520624517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.520653007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.528977515Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.529011436Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.52919782Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.529317033Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.561043338Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.561085969Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.56109399Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.56110159Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.56110807Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.561197002Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"354ddbad-57e0-4ee8-a517-6a2cd02e31fe","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":103237,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.561407547Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"106.808279ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"354ddbad-57e0-4ee8-a517-6a2cd02e31fe\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"354ddbad-57e0-4ee8-a517-6a2cd02e31fe\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.669598178Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.670007947Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.670018717Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.679107182Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.679142913Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.688048114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.688071764Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.697370324Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.697394495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.707023902Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.707047703Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.716583748Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.716611238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.7251056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.725136451Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.732978388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.733002438Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.7437218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.743749031Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.743955525Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.744079658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776194843Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776231513Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776237544Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776242634Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776246594Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.776311235Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"09598470-14a6-4bc4-9fe5-abcdc38a390e","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized)]","latency_us":106509,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:07.776454018Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"135.327151ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09598470-14a6-4bc4-9fe5-abcdc38a390e\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"09598470-14a6-4bc4-9fe5-abcdc38a390e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.913596903Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.91392652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.91393517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.924210002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.924231353Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.933329688Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.933355418Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.942056835Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.942087686Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.951546399Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.951570489Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.960487351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.960507921Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.969090565Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.969116445Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.97774427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.977765971Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.986833905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.986873666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.98703686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:07.987150282Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.020970126Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.021065345Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.021071879Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.021078395Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.021086092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.021192257Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a51e5007-4cf3-4a06-a937-809e39943b2b","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":107394,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.02137858Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"140.193345ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a51e5007-4cf3-4a06-a937-809e39943b2b\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a51e5007-4cf3-4a06-a937-809e39943b2b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.162888005Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.163101179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.163109307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.173492301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.173521327Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.182327924Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.182350856Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.191328174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.191351208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.20128053Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.201303069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.210379157Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.210408108Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.218837116Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.218860244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.226698745Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.226722296Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.235405201Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.235428199Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.235667025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.235758114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268005306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268029203Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268034294Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268053306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268058144Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.268133943Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f319b728-3266-43ec-a3d8-9a8c32d55c46","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":104856,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.268275743Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"110.508456ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f319b728-3266-43ec-a3d8-9a8c32d55c46\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f319b728-3266-43ec-a3d8-9a8c32d55c46\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.379710936Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.379919797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.379926219Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.38928629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.389313854Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.398281708Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.39830614Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.407101058Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.407124083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.416259096Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.41628033Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.424974718Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.424997191Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.434030834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.434053015Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.442926419Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.442949209Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.451887548Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.451914005Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.452056902Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.452138063Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.482835225Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.482887436Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.482897599Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.482903074Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.482908277Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.483006894Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"94311f6d-5426-43a2-9bf4-7f0285639de0","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":102896,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.483190564Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"163.458329ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"94311f6d-5426-43a2-9bf4-7f0285639de0\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"94311f6d-5426-43a2-9bf4-7f0285639de0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.648082183Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.648393394Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.648402075Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.658683821Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.658774976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.6672968Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.667319319Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.67664069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.676659206Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.686110243Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.686138378Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.695224337Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.695252454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.70377447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.703800655Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.711710957Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.711736373Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.720445644Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.720480745Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.720659305Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.720770917Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.752887227Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.752916478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.752921887Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.75292664Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.752929847Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:08.753062026Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a36ad0ef-77f5-480e-aa46-4a23e6076b53","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized)]","latency_us":104513,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:08.753174135Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"329.577955ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a36ad0ef-77f5-480e-aa46-4a23e6076b53\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a36ad0ef-77f5-480e-aa46-4a23e6076b53\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.085264378Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.085463432Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.085471364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.095865841Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.095956168Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.105599671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.105619969Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.115054279Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.115074558Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.124403718Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.124426181Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.133439904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.133459282Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.142322494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.142344104Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.151643556Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.151671232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.160576856Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.160604222Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.160782566Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.160873731Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.193382502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.193408302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.193416712Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.193423003Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.193429191Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.19353738Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1e62cc4d-2602-4818-9b05-572c20da003e","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized)]","latency_us":107876,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.19371115Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"446.785627ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e62cc4d-2602-4818-9b05-572c20da003e\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e62cc4d-2602-4818-9b05-572c20da003e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.642462012Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.642747655Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.642756693Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.652321107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.652351969Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.661397537Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.661492856Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.67109503Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.671119274Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.68023738Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.680261305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.689714002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.68974133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.698808641Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.698832772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.708285643Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.708318185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.717476711Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.717512093Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.717734047Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.717818696Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750144978Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750170404Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750175514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750181326Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750186183Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:09.750265056Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ede522c7-bda4-4fa0-8c00-e4c0b37085c3","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":107414,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:09.750411168Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"323.626658ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ede522c7-bda4-4fa0-8c00-e4c0b37085c3\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ede522c7-bda4-4fa0-8c00-e4c0b37085c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.075502199Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.075853749Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.07586349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.085520558Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.085538418Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.095087934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.095105531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.10399415Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.104012572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.113206937Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.113223653Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.122446196Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.122462649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.132134671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.132156075Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.141185267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.141206305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.149950106Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.149975438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.150142842Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.15024099Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182328056Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182382366Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182391366Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182397825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182402298Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.182488972Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6937f18b-0f0d-4f0c-bc89-78e14e72ff36","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized)]","latency_us":106536,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.182656301Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"282.004308ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6937f18b-0f0d-4f0c-bc89-78e14e72ff36\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6937f18b-0f0d-4f0c-bc89-78e14e72ff36\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.467027757Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.46732511Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.467335038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.477059284Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.47711906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.486015716Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.486048502Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.495405635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.495443051Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.504258424Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.504289568Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.513095012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.513127581Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.522441791Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.522480023Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.531840089Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.531869546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.54097068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.541014331Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.541194682Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.541307963Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573686243Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573713131Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573718035Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573723838Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573729753Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:10.573804802Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"525d7cb0-2b68-4d3c-b26a-ed2ad648d29a","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":106366,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.573997981Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"606.890237ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"525d7cb0-2b68-4d3c-b26a-ed2ad648d29a\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"525d7cb0-2b68-4d3c-b26a-ed2ad648d29a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:10.649063503Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"144.119036ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.18383701Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.184104596Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.184123234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.194994931Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.195022223Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.204690842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.204724087Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.21493946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.214965364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.224102648Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.224129602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.233914335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.233935927Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.242991474Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.243016591Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.252304912Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.252335493Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.261403323Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.261440131Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.261615373Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.261717112Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.293670481Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.293720207Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.293730866Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.293739107Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.29375033Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:11.293946574Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"767e6100-54f0-45df-91aa-789c27fb40ad","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":109670,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:11.29408077Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"890.300188ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"767e6100-54f0-45df-91aa-789c27fb40ad\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"767e6100-54f0-45df-91aa-789c27fb40ad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.187379783Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.18761003Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.187615889Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.198225184Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.19826245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.207487594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.207515297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.217502154Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.21753132Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.227838416Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.227873433Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.236278618Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.23629943Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.244132853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.244155044Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.253560577Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.253581681Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.262011629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.262048867Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.262230644Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.262345658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.293021232Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.293051354Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.293057889Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.29306432Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.293072299Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:12.293221795Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"7c43a2ca-bd3c-4151-a368-744d8edacee6","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":105457,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:12.293397197Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"788.922433ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c43a2ca-bd3c-4151-a368-744d8edacee6\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c43a2ca-bd3c-4151-a368-744d8edacee6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.085783983Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.086154176Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.086163486Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.096325735Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.096358136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.106149065Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.106175869Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.114744072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.114772826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.124463707Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.124504329Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.134940584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.135030563Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.143792609Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.143823248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.152647494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.152673865Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.161090977Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.161137046Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.161306972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.161424022Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.193987556Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.194014275Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.194018419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.194024194Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.194028994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:13.194202113Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"18ce4a38-dc1f-426a-ab53-5d83535efcfe","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":107899,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:13.194343407Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"883.858118ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18ce4a38-dc1f-426a-ab53-5d83535efcfe\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"18ce4a38-dc1f-426a-ab53-5d83535efcfe\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.082156381Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.082387002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.082406727Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.093139974Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.093176068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.103425888Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.103447948Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.112565276Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.112593843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.122040433Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.122068662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.131395426Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.13141547Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.140229772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.140251663Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.149008057Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.149027801Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.157687604Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.157717276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.157960137Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.15805584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.189725572Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.189747612Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.189755498Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.189761901Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.189767929Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.189911153Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5d6246c1-19ca-4eef-a895-3e4c4fb990c8","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":107373,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.190126371Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"579.263537ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d6246c1-19ca-4eef-a895-3e4c4fb990c8\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d6246c1-19ca-4eef-a895-3e4c4fb990c8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.772245174Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.772827709Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.772849113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.784336165Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.784383679Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.794381133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.794419712Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.803488425Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.803523685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.813655707Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.813752142Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.823306496Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.823345487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.833537438Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.833573186Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.842600911Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.842631821Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.851923997Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.85196642Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.852140921Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.852249626Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.883926308Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.883952811Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.883973212Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.883979212Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.883984978Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:14.884069017Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"4cf79680-649f-4cce-a10e-f9e72a26899c","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized)]","latency_us":111242,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:14.884193239Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.078141043s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4cf79680-649f-4cce-a10e-f9e72a26899c\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4cf79680-649f-4cce-a10e-f9e72a26899c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.965767728Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.966135503Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.966145544Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.976606403Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.976655894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.986184587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.986206338Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.995460365Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:15.995494041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.004680695Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.004703084Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.014140371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.014160171Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.022744861Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.022774111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.031749746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.031777412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.041516183Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.041550759Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.041717319Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.0418387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073098023Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073235102Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073241711Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073248461Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073253656Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.073360589Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ef15edb3-dad2-4bef-a40e-e85f0f17fab1","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":107102,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:16.073509533Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.589952466s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef15edb3-dad2-4bef-a40e-e85f0f17fab1\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef15edb3-dad2-4bef-a40e-e85f0f17fab1\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.390463673Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57358"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:16.390639327Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"011ec414-e1bb-4cd8-b739-bd43cf8f999a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57358","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.668206777Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.66856227Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.668572348Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.678762798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.678795949Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.687837533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.687880884Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.697234529Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.697276004Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.706320422Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.706353159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.715376762Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.715408057Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.72492767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.724957192Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.735160586Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.735203533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.744264052Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.744302134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.744465676Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.744583925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777393788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777423676Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777430792Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777437917Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777443449Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:17.777575292Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"da9ae5e9-3c4a-438d-ae82-aba28887edfa","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized)]","latency_us":108933,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:17.777835946Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.284195642s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da9ae5e9-3c4a-438d-ae82-aba28887edfa\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da9ae5e9-3c4a-438d-ae82-aba28887edfa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.069298475Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.069555505Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.06956189Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.0802288Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.080256804Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.090467391Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.090488532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.099630929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.099652774Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.109115739Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.109150286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.118118159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.118143116Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.127395286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.127417234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.136391335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.136410638Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.145571779Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.145662607Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.145798585Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.145894343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.178318203Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.178376995Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.178387363Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.178398549Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.17840767Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:20.178509841Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"7faab4b7-2e9c-4601-846a-a8445b9103de","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":108755,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:20.178667139Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.248395824s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7faab4b7-2e9c-4601-846a-a8445b9103de\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7faab4b7-2e9c-4601-846a-a8445b9103de\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.431856561Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.432203575Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.43221774Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.442854495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.442886933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.451873582Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.451906358Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.461740598Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.461779533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.471397497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.471425032Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.48025622Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.480288059Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.490780214Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.490818512Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.50028795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.500323735Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.509303595Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.509363053Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.50956117Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.509691307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542247473Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542272186Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542278205Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542282977Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542288668Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:21.542383638Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"faf6b898-8f7d-444f-9473-3586877bda52","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":110082,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:21.542514498Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"3.661894502s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"faf6b898-8f7d-444f-9473-3586877bda52\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"faf6b898-8f7d-444f-9473-3586877bda52\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.214983113Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.215274719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.215284469Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.226509812Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.226537965Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.237090481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.237108088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.24611914Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.246138622Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.255008687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.255029753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.263379Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.263400817Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.271947617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.27196587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.280249775Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.280266923Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.28779069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.287846875Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.288098305Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.288224195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.319724197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.319763488Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.319771129Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.319777457Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.319785154Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:25.319907136Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"78393163-7566-4a02-be23-3a00344e0bb9","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized)]","latency_us":104554,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.320132927Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"4.413833981s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78393163-7566-4a02-be23-3a00344e0bb9\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78393163-7566-4a02-be23-3a00344e0bb9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:25.872102123Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"721.970352ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.747198341Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.747509421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.747523427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.758684904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.75871315Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.768935652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.768963112Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.77806906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.778094505Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.788267883Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.788300969Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.7973683Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.797399538Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.806040537Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.80607034Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.813937078Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.813975329Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.823412244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.823453261Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.82367297Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.823799994Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.855871729Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.855894052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.855899893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.855904936Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.855910861Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:29.856002926Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"fb70c5da-6141-4dff-b6f9-412433bf23df","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":108406,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:29.856133589Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"4.469001235s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb70c5da-6141-4dff-b6f9-412433bf23df\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb70c5da-6141-4dff-b6f9-412433bf23df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.338843171Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.339047588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.339055228Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.349813301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.349831629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.359353408Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.359368501Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.36888716Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.368908704Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.377503956Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.377518853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.386155893Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.38617049Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.394715342Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.394729676Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.40329321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.403307207Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.412778467Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.412805506Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.412975562Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.413080377Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.444983523Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.445016355Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.445023893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.445029902Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.445040562Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:34.44516738Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"733d6909-a384-4709-93ad-0d5d58538d47","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":105934,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:34.445328953Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"5.005443877s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"733d6909-a384-4709-93ad-0d5d58538d47\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"733d6909-a384-4709-93ad-0d5d58538d47\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.465216839Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.465607861Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.465621342Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.476412104Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.476437062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.485674697Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.485693092Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.495530083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.495548345Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.504303242Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.504321318Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.512355268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.512372772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.522687452Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.522712127Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.531359454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.531379301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.540837872Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.540866906Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.54110634Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.541231546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.574681638Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.574714788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.57472151Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.57472751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.574732273Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:39.574812637Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6fb89a9a-e510-419b-9a69-3b05fe0f563b","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized)]","latency_us":109087,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:39.574961468Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"7.339668386s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6fb89a9a-e510-419b-9a69-3b05fe0f563b\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6fb89a9a-e510-419b-9a69-3b05fe0f563b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:41.671660918Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.459463926s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.50389445Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.503983251Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"eff8b57b-7f94-4cc8-a973-cae7abf5e46d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62106","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.935597911Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.93580141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.935810522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.947154279Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.947184439Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.957334799Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.957352546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.966409089Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.966431007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.97603882Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.976056081Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.985038572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.98505557Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.994355057Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:46.994372233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.00357478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.003593944Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.012995814Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.013039596Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.01322142Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.013337504Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.047307557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.047346117Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.047353917Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.04736197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.047368092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:47.047507332Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"865bdfed-4b2f-43a1-b341-02bfc075ab79","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized)]","latency_us":111564,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:47.047693787Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"6.717131566s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"865bdfed-4b2f-43a1-b341-02bfc075ab79\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"865bdfed-4b2f-43a1-b341-02bfc075ab79\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.784225683Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.784578265Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.784586928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.795768423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.795803561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.805874232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.805896217Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.816225583Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.816257627Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.826491426Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.826513982Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.835890669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.835922057Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.845114286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.845136908Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.854803297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.854824701Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.864148884Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.86418241Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.864456886Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.864585776Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.895763714Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.89578852Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.895799311Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.895805134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.895810759Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:00:53.895911936Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f69d6890-a08b-4b1f-bada-4bb9f23505d9","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":111233,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:53.896043948Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"8.342733235s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f69d6890-a08b-4b1f-bada-4bb9f23505d9\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:00:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f69d6890-a08b-4b1f-bada-4bb9f23505d9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:00:58.212117399Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.80938236s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.261860059Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.262184995Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.26219617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.273915275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.273938507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.284170136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.28419138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.29378831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.293807238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.302875138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.302895595Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.311597056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.311616922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.320452603Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.320472066Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.329374602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.329400271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.339160739Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.339262863Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.339452401Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.339564642Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373158215Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373195754Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373204051Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373210144Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373217701Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:02.373381677Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"15e4cd51-322a-4993-b683-8347431b19be","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":111044,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:02.373527302Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"7.115610861s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15e4cd51-322a-4993-b683-8347431b19be\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"15e4cd51-322a-4993-b683-8347431b19be\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.509928659Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.510236334Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.510245926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.521185602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.521207456Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.531555662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.531571723Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.541939704Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.541960095Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.553146048Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.553165033Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.562409728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.562428198Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.571388141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.571406799Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.580626208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.580644313Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.589737515Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.589863715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.590088504Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.590232133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623556673Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623586102Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623590771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623594212Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623599087Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:09.623704287Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"b428e719-0299-4dc1-8a3c-1507d29e11f4","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized)]","latency_us":113353,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:09.623840892Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"6.237279287s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b428e719-0299-4dc1-8a3c-1507d29e11f4\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b428e719-0299-4dc1-8a3c-1507d29e11f4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.878759095Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.87903241Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.879039666Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.889712057Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.8897408Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.899836238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.899853498Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.908469233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.908486305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.9161633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.916182107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.925383971Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.925400396Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.934502744Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.934519497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.942947853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.942964071Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.951534032Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.951567445Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.951755043Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.951865249Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.982633437Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.9826784Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.982686463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.982692904Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.982699307Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:15.982819197Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"866dcb94-d697-4180-8358-12b275d31372","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":103652,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:15.98300537Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"13.841341282s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"866dcb94-d697-4180-8358-12b275d31372\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"866dcb94-d697-4180-8358-12b275d31372\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:16.105587923Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"5.308627776s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:16.618191722Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:16.618315681Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"85983449-55c7-4aab-a934-9106137bcea7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42751","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.863410365Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.863783866Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.863795453Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.87517556Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.875206189Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.885671719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.885695531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.896093411Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.896121481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.905668147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.90569539Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.914752518Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.91477649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.92390267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.923926698Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.933120362Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.933142618Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.942701122Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.942737601Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.943062456Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.943183977Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.976950301Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.977050704Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.977057163Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.977061954Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.977067672Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:29.977144315Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"22255d59-4927-4d7c-a83b-e4b96d980557","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized)]","latency_us":113250,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:29.977288168Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"15.837031335s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22255d59-4927-4d7c-a83b-e4b96d980557\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22255d59-4927-4d7c-a83b-e4b96d980557\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:36.503984393Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"8.604642813s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.858261683Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.858507867Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.858517645Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.869802927Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.869826411Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.879435664Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.879463507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.889312557Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.889340194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.89831834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.898341151Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.907524461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.907545068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.916792266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.916810669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.926598531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.926615772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.935368381Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.935411067Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.935694461Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.935823651Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.96864515Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.968676201Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.968682707Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.96868881Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.968696517Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:45.968815451Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"2642247d-5658-4190-a5a9-263d38dff579","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized), (\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized)]","latency_us":110161,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:45.968985864Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"10.196951695s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2642247d-5658-4190-a5a9-263d38dff579\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2642247d-5658-4190-a5a9-263d38dff579\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:46.732838882Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:46.732974279Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"07e0a952-851b-40b7-abaf-05e265d8e6ca","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45838","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.195637437Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.196045067Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.196057592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.208505083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.208547196Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.218629466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.218657131Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.22846419Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.228490402Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.237604658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.237635869Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.246625498Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.246654702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.256985826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.257024349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.266274252Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.266300108Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.274966248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.275020503Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.275346028Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.275479062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.309392929Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.309414238Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.309419348Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.309424092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.309430064Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3340","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:01:56.309550433Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"07202b4a-9912-4552-abda-8e538d32b67c","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\", TimeNotSynchronized), (\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\", TimeNotSynchronized), (\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\", TimeNotSynchronized), (\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\", TimeNotSynchronized), (\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\", TimeNotSynchronized)]","latency_us":113414,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:01:56.30969826Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"13.122994761s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07202b4a-9912-4552-abda-8e538d32b67c\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:01:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07202b4a-9912-4552-abda-8e538d32b67c\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:00.208289006Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"23.444939252s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.469077407Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.469273049Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.469279706Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.481201231Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.481225062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.491804321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.491825201Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.501565738Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.501587349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.511587692Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.511608148Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.5217034Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.521724428Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.531761393Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.531782571Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.54152147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.541543127Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.550787855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.550823106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.551052029Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.55122427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.635466092Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.635516943Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.63552328Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.635528887Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.635533987Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.731251794Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.735420686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.739663755Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.744427033Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.749167166Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.75419746Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.759095235Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.76399245Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.768364747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:09.772918632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.709066167Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.709216966Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.709282856Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.715150254Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.73635674Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.736541835Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.736560154Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.742493507Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.781569181Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:12.781717449Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 40.739463ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.858929757Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 57.570157ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.867921642Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 53.26913ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.877450369Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 59.223501ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.888513866Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 80.387977ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.907880308Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 54.032147ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:12.933185828Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 79.41118ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:13.042705497Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 136.271891ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:13.173494465Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 267.016672ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:13.327483207Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 107.04401ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:13.511469192Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 81.276791ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:13.875025021Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 293.65643ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.015350101Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 163.254075ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.086218105Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 237.054184ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.131098694Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 206.178084ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.167258091Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 122.627461ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.200097583Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 390.57941ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.236744758Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 387.936852ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.270594425Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 592.574036ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.347467193Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 478.591719ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.38976248Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 571.645669ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.408800098Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 522.153323ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.654006189Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 859.434289ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.686310786Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.150255079s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.905264117Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 720.57356ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:14.961838361Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 704.791021ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:15.002936023Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.323408384s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:15.190611368Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 781.592131ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:15.559878803Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.02111164s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:15.700669838Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 1.301627655s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:15.722870012Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.394395802s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:16.072671132Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 617.630744ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:16.354116484Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.453024749s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:16.5254993Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.27546435s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:16.732982027Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:16.848745271Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:16.848833501Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f10b05ed-ab04-45c2-bb96-a2bb03d45cb7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64214","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 934.597093ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:16.982700216Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 784.838366ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:17.039706944Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.065830632s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:17.485702338Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 651.34153ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:17.837864588Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.041979943s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:17.929899805Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.228756196s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:17.99090713Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.238048461s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:18.024324179Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 1.07836011s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:18.499947955Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.374140947s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:18.562290859Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.413306186s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:18.985357591Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.417912942s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:19.233861742Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.136209795s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:19.273790743Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 811.027454ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:19.593075594Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 506.053518ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:19.950429521Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 914.508671ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:20.412592849Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6\") failed: Property not found. retry in 1.485654572s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:20.421105394Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 747.804663ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:20.428837683Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.455038037s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:20.467100912Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.06412489s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:20.665685992Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.279341174s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:21.188058201Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 685.556184ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:21.339429083Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 637.579653ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:21.742218295Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.207631481s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:21.935381559Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.037325837s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:22.03729326Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.171679347s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:22.392620842Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.267897102s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:22.48095994Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 552.05489ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.091196601Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 646.169853ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.160640768Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.121436873s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.577069033Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.193227845s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.652832893Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 534.570452ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.759954479Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.270960144s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:23.816313611Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 960.291718ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:24.307324279Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 979.588598ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:24.710606668Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 661.015944ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:24.857053732Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 665.05582ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:25.09717247Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.288045032s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:25.276211289Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.056160784s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:25.528602844Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.001787037s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:25.699638523Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 646.485868ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:25.771987556Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 501.621258ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:26.427297361Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 574.399259ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:26.573209293Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.24124288s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:26.592947659Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.186932868s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:26.709738664Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 773.811755ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:26.93735087Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.364422876s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:27.157284162Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 1.104679482s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:27.721528783Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 548.270746ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:27.843120307Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.387043582s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:27.903868228Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 1.493758056s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:28.400147925Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 688.886758ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:28.530462005Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 972.728915ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:28.835182121Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.263843487s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:29.227624675Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 554.699957ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:29.299291473Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 812.378741ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:29.816508732Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 832.019963ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:29.863006041Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 829.820951ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:29.902147295Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6\") failed: Property not found. retry in 1.002716247s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:30.500401393Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385\") failed: Property not found. retry in 863.554997ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:30.637535637Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c\") failed: Property not found. retry in 1.065411443s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:30.703515425Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98\") failed: Property not found. retry in 551.420689ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:30.741170895Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.789880845Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.789963667Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_cockroachdb_db836525-f5a5-47c1-a1d6-2213d0a84c98"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.789977937Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_cockroachdb_e0eb4a31-ef27-4215-82a7-e26050aa5385"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.789990957Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_cockroachdb_bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.789996408Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_cockroachdb_f70b01ef-9449-46a4-bea0-2ce69b7051a6"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.790002018Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.790376756Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.790741054Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6d505522-87b7-4866-8717-9eb1614ed18b","remote_addr":"[fd00:1122:3344:101::1]:34275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":22285243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.823766259Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:31.849606953Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"afb0db00-c530-4fc8-a193-cef4b598faec","remote_addr":"[fd00:1122:3344:101::1]:54697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_ad212657-76bb-4c34-a6a1-fcbcd07fe95c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25690,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:02:31.849911909Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"146.546814ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afb0db00-c530-4fc8-a193-cef4b598faec\", \"content-length\": \"124\", \"date\": \"Thu, 09 Jan 2025 01:02:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afb0db00-c530-4fc8-a193-cef4b598faec\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:32.029995162Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3769"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:02:49.03908526Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3787"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.592038532Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.594801074Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"55622f14-7c54-4b0c-a910-4e75f2cbc835","remote_addr":"[fd00:1122:3344:101::1]:54697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33599129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.600346919Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56677"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:05.619205634Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"22.725917848s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: b7e5d812-3b66-48ae-9425-d561b75ab23b (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6135d2f0-37b8-443e-a741-5ec4e46a4ef0 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a98745ae-2f0a-4dbf-9d35-ec0663205372 (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 772a58ca-731f-4866-9630-9630892160a5 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 840e24c2-515b-43f7-b977-c352662aa532, kind: Service { id: 772a58ca-731f-4866-9630-9630892160a5 }, name: Name(\"ntp-772a58ca-731f-4866-9630-9630892160a5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 162, 151])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bbc2cdc2-f11f-4af6-bd01-4d296f3e62e6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f70b01ef-9449-46a4-bea0-2ce69b7051a6 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad212657-76bb-4c34-a6a1-fcbcd07fe95c (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0eb4a31-ef27-4215-82a7-e26050aa5385 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: db836525-f5a5-47c1-a1d6-2213d0a84c98 (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }, OmicronZoneConfig { id: 870344ce-2728-444f-8d5a-e3771e15477e (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 71878588-3bc5-4be9-bddb-893f09285184 (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5236fac0-d91c-403e-8658-32167d628d4d (service), filesystem_pool: Some(ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: cffb0aad-1932-4a44-891c-2739991d1934 (service), filesystem_pool: Some(ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d369b97b-e309-4e30-bf3f-1e4b78ee9699 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: f6df9c4a-6f0d-4eae-894a-405b9001ae75 (service), filesystem_pool: Some(ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 9ec1ec86-e462-408e-bb05-5b443483f792 (service), filesystem_pool: Some(ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 608978bb-69bf-4b80-bacf-2e812b610352 (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de32ba2-a881-47b5-acf6-8242e72b57bc (service), filesystem_pool: Some(ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External } } } }, OmicronZoneConfig { id: 415ee469-d5a2-4856-93b1-6e5d1fb10bbb (service), filesystem_pool: Some(ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5dda8bfa-5424-45e5-8461-826a4ee1efab (service), filesystem_pool: Some(ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External } } } }, OmicronZoneConfig { id: c8663988-f19e-423f-a69e-a5c3de3a9f25 (service), filesystem_pool: Some(ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0501a4ce-6961-4f31-96f9-d2267ec97f3a (service), filesystem_pool: Some(ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External } } } }, OmicronZoneConfig { id: 103b3e3e-27bb-47c7-89b9-15a3a8f13a1e (service), filesystem_pool: Some(ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External } } } }, OmicronZoneConfig { id: 788d2519-7437-4048-92bd-a1a85eea6416 (service), filesystem_pool: Some(ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aecbbe1-3a43-407e-bceb-6b3134c30e9f (service), filesystem_pool: Some(ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 463d55a1-4575-4d27-b564-f1c352fed8fb (service), filesystem_pool: Some(ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.663837901Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.665234283Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46418"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.667584476Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.667597906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.682753598Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.682801269Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.693245835Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.693299566Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.703974586Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.704026498Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.713472381Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.713514472Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.722356821Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.722396822Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.73118685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.731244522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.741134215Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.741180266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.750365673Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.750403234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.770416705Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.770469786Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.783264915Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.783310136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.794305054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.794345195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.808894423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.808955135Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.818355857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.818395958Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.834844889Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.837170831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.844738862Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.844755022Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.855064594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.855111133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.865258405Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.865308456Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.875999947Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.876050638Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.886126285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.886168216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.898356066Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.898699592Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.899497717Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:05.900009979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.148869582Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.150507239Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.150538757Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.150561858Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.167200946Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.167472952Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.167496162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.167512303Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.192629029Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.1926642Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.392148472Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.39256115Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.392592135Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.394115244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.44005563Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.440344753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.440372776Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.440404493Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.441709788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Configuring new Omicron zone: oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.442448835Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.447188691Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.452502591Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.458914766Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.464022931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.468654626Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.474201766Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.479599099Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.609584655Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.61513376Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.620546862Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.6262286Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.63108254Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.636231926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.640785869Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.645434843Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.651244874Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.656461692Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.661533637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.667545222Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.673472627Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.679688456Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.755197679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.756321005Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.775807414Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.786993627Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.796379768Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.814231371Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.828639098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.899438783Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:06.968815658Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.031593574Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.095380423Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.100736184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.100763624Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.158670791Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 246, 106]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.158697121Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.160302957Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.160314618Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.218384678Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 203, 193]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.218409738Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.223261168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.223282548Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.283535515Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 188, 234]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.283563893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.285506802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.285540442Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.344189396Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 251, 54]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.344222706Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.346528018Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.346547378Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.405746674Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 183, 156]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.405778265Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.40603458Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.40604918Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406103552Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406114402Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406159793Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406167733Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406213654Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.406225175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.407539014Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.407554205Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.435169288Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.436589929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.496622629Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.505784914Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.505976422Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.512700426Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.512719217Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.51641278Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.516648935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.720985545Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.773485069Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.794661327Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.815411945Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.833693697Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.84801996Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.86361426Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.877809842Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.891611774Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.900832441Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:07.909883656Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.326731793Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.344196587Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.344241868Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.373489638Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.39620714Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.401993751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.402064412Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.419526036Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.420202272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.42192268Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.423634419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.425401549Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.430566225Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.430659647Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.445765578Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.459224152Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.469940494Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.482435196Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.495221684Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.506789845Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.881649293Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:21.881836477Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 59.383624ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:21.996796841Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 41.088572ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.006318685Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 59.041335ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.016678419Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 68.005652ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.030087202Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 55.547906ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.041859577Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 43.660834ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.054132154Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 45.685379ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.064712843Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 64.179586ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.073610624Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 96.086443ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.083942227Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 51.782986ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.094595497Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 101.441787ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.105530074Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 140.928493ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.118449725Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 38.778966ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.131327476Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 80.019288ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.14259108Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 57.086286ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.171500932Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 89.105869ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.207398832Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 32.462777ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.230482483Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 95.480386ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.281258319Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 83.312615ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.30436059Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 185.318908ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.320471544Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 143.169878ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.341713543Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 293.094602ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.361943999Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 281.77551ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.380502058Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 65.935885ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.399618269Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:22.40055146Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 204.234267ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.418486985Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 143.481563ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.437810701Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 143.951582ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.453791211Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 103.605224ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.468581495Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 150.196884ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.485323163Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 238.325167ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.49807472Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 249.779232ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.513505389Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 106.454979ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.528381384Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:22.528488567Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 432.928162ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.553847679Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 209.129933ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.589896422Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 258.228384ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.608144614Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 436.693108ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.626815485Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 496.539499ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.643240796Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 482.821151ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.661456267Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 528.359467ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.676710821Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 224.598739ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.690198605Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 494.943284ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.703869594Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 317.856889ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.752318007Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 49.792424ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.762824644Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 404.340724ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.776233316Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 237.710024ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.812966175Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 56.985713ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.823603025Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 844.242954ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.879716501Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 231.953363ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.892363507Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 641.94734ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.929370902Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 962.723568ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:22.99978782Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.123554725s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.067514148Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.155336407s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.08222714Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 747.074214ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.098285882Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 392.048154ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.144442634Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 730.719374ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.161246683Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.15328058s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.180821065Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 423.139441ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.197572883Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 641.78182ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.21649586Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.053254122s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.235658932Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 834.101071ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.580962313Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.221355794s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.604000692Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 764.962155ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.639453752Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 1.498616776s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.740109373Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.027765412s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.862819752Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 977.765471ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.878766982Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 1.035864906s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.907884459Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.183528644s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:23.979918104Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 548.273046ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.283314709Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.054276665s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.3241394Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 604.438956ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.345573004Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.039597338s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.362987117Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.164910143s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.4207632Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 842.36799ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.434281105Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.47173059Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.471993986Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.506884583Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.507013036Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.507118719Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.515388555Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.599426481Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.599630496Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 65.920239ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.742824547Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 66.865582ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.757748814Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 30.344779ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.788787844Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.790950763Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:24.792063188Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 56.808168ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.822493174Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 111.061589ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.835624311Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 75.010875ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.846406824Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 712.051905ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.864576964Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.093821139s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.922987191Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 1.184385713s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:24.976505239Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.316553839s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.045730501Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 702.52308ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.060718259Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 168.958082ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.077460427Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 266.328485ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.089875947Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 183.767365ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.10065Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.340770425s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.112392345Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 49.940603ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.123120327Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 61.222622ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.143113558Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.213851072s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.175587291Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 98.555344ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.189573656Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 132.532777ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.21588321Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 685.73792ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.251124985Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 256.233644ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.262839809Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 804.067513ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.289164313Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 268.969538ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.300237973Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 227.797998ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.311321163Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 154.134781ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.362985029Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 532.978966ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.373835064Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 502.91767ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.389907396Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 820.189287ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.414224355Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 589.996487ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.532310159Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.039171035s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.542888728Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 387.383989ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.553227531Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 954.374885ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.581324235Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 765.123415ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.592678661Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 719.825872ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.604190121Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 526.031748ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.779004485Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.190871536s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:25.966817263Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 961.790555ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.016169506Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 784.288496ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.038158762Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 1.12934802s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.058121013Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.036037307s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.080147839Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:26.081325226Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:26.081616223Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 940.101288ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.108627622Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 952.935215ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.13909841Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 746.7205ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.200971896Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.488032314s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.25302257Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 1.292889351s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.322899616Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.286326528s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.343198125Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 949.984506ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.373967239Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.399624208s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.388848935Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.078429774s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.403800742Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 61.972567ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.417528431Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.375185959s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.473689429Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 129.370294ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.500491473Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 565.459802ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.550810139Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 857.604729ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.602493525Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 187.26092ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.653422584Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 760.234143ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.842096191Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 223.315885ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.864589158Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 1.254744213s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.972211126Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 585.404248ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:26.998715934Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 1.451265504s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.070300259Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 549.79841ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.10710412Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 1.378461798s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.128250127Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 1.173010895s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.146097969Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.171372066s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.167007261Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.46106665s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.184130818Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 596.200712ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.212128689Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.042029001s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.345995359Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.258386801s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.476837331Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 974.336801ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.497021317Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 1.331164137s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.607465469Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 1.038763576s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.634626302Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 828.302596ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.656303011Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.041074494s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.685599052Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 1.157831365s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.722189907Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.333537748s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.795861029Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 573.962576ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.822860718Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 1.277262143s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.845768015Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 724.321344ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:27.872922228Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 637.474953ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.249028774Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 1.227640537s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.343074186Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.360293699s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.366714449Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.499260755s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.414219431Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 807.468774ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.437408604Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.235666515s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.488066477Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 770.262535ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.504043127Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 1.388335259s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.522990675Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 934.361694ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.538687989Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 740.758489ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.613823694Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.008689721s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.659378912Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 1.121494869s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.686348171Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.081553168s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.74039872Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 732.753112ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.752333749Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 1.313055518s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.893646808Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 1.425994727s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.905583947Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 1.118379156s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:28.949536759Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 1.02541699s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.138217796Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.087374948s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.152556929Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 650.110119ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.259000631Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 1.324381316s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.287710779Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 775.26939ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.367408337Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 1.155049266s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.485820798Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.44114889s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.497210986Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 553.53976ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.581149519Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 507.452954ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.682351733Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 957.951171ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.736656478Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 811.520129ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.746155152Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 632.284703ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.820689744Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.003515751s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.835090629Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 724.538667ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.921426497Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.361978458s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.933495619Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 918.384786ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:29.945074Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 772.969807ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.08067759Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 1.056157852s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.148539631Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.34844018s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.162574217Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 1.301967612s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.176925801Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.397752622s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.203710095Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 1.261593295s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.220855052Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 710.890643ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.251297329Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 652.226557ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.346049787Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 847.119286ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.468180222Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.194819015s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.571142355Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 779.947625ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.63518741Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 1.289193258s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.654970356Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.098269865s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.668076602Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.044980707s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.710032899Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 692.899558ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.851459069Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 540.483705ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.868367101Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 1.306675201s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.88381459Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.022561283s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.95341635Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 572.836896ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:30.976638174Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 1.470893307s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.017339292Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 1.189842411s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.220211279Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 611.12928ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.307873647Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 1.443225705s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.3310503Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 577.475826ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.421980751Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 660.34724ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.435088197Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 807.858438ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.495465799Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 1.182711612s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.529215591Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 855.894864ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.567083225Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.456972521s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.586490463Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.247236492s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.63333185Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.350981383s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.682151565Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 590.824528ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.820870221Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.255124846s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.836420872Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 546.424305ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.856891233Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.008057363s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.93697264Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 627.710784ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.962821934Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.463352546s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:31.991659324Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 799.614597ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.01584669Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:32.082393751Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:32.121706568Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9dbdb836-595a-4733-9a1f-d60c0e4f3025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52104","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":14805,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 1.338203844s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.138283182Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 714.182267ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.205519881Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 564.321596ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.315121782Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:32.405803457Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 1.413994977s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.466050186Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:32.466128168Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.213149797s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.486756913Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.397444724s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.499219715Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 905.867473ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.51231108Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 642.019579ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.523806229Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 922.31169ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.605033372Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 59.541776ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.702562142Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 992.448094ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.723540725Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 132.149219ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.772442199Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 672.061687ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.786342852Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 1.089584858s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.827381858Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 687.415251ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.890871501Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.222045826s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.900550179Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 236.183664ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.914124765Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 983.463474ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.930647858Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 578.396107ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:32.961276329Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 910.442611ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.045205753Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.347282794s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.054805139Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 1.141928515s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.107681952Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 261.03958ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.162172732Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 587.561752ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.175955703Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 543.829481ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.429357269Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 642.432651ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.438658209Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 1.083851851s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.472650956Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 1.495902549s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.483835079Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 513.729956ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.493322513Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 1.186371222s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.536707782Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.022067886s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.549553731Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 641.567883ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.587586379Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.195352406s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.708990409Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 877.827263ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.725071931Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.248542421s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.772570133Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 762.554739ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.88902761Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 557.825214ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.90631597Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 941.788075ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.9222499Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 588.505909ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.933436082Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 1.165020629s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.964981834Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 1.025462371s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:33.982807516Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 1.080194681s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.017220403Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 1.345964378s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.090136538Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792\") failed: Property not found. retry in 1.118706114s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.131295456Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc\") failed: Property not found. retry in 1.240558067s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.238859183Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25\") failed: Property not found. retry in 927.010244ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.263483919Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab\") failed: Property not found. retry in 1.033267843s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.411884477Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb\") failed: Property not found. retry in 1.478667121s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.478652713Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75\") failed: Property not found. retry in 1.09011055s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.533375628Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 527.822666ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.572611823Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699\") failed: Property not found. retry in 1.381651833s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.585138956Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 1.072300021s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.614559219Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934\") failed: Property not found. retry in 778.00282ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.66159467Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184\") failed: Property not found. retry in 1.352916846s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.732682054Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e\") failed: Property not found. retry in 864.422166ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.875060447Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a\") failed: Property not found. retry in 1.352441878s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.916516172Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352\") failed: Property not found. retry in 863.374623ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:34.991209417Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416\") failed: Property not found. retry in 555.32064ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.025193084Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f\") failed: Property not found. retry in 1.200278808s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.042208158Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e\") failed: Property not found. retry in 793.516622ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.118470738Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d\") failed: Property not found. retry in 994.739274ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.133807334Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb\") failed: Property not found. retry in 788.635973ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.146411659Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 557.654043ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.452400642Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327\") failed: Property not found. retry in 970.518403ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:35.709907772Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 1.027088869s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:36.055834966Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 931.113528ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:37.11447135Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 1.326358666s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:38.069305103Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:38.619342822Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"61.961359025s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:39.006883374Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:03:39.009410072Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:39.247845681Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"wait for service Some(\"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6\") failed: Property not found. retry in 1.104617731s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:39.407527084Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:03:40.014622751Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546667254Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_103b3e3e-27bb-47c7-89b9-15a3a8f13a1e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546749365Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_pantry_f6df9c4a-6f0d-4eae-894a-405b9001ae75"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546758686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_463d55a1-4575-4d27-b564-f1c352fed8fb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546763276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_nexus_5236fac0-d91c-403e-8658-32167d628d4d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546768176Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_oximeter_d369b97b-e309-4e30-bf3f-1e4b78ee9699"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546785906Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_pantry_608978bb-69bf-4b80-bacf-2e812b610352"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546790477Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_5dda8bfa-5424-45e5-8461-826a4ee1efab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546794666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_nexus_cffb0aad-1932-4a44-891c-2739991d1934"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546801867Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_clickhouse_56b59c3e-b7da-4d87-9b2c-5c12d0a9b9c6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546807807Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_0501a4ce-6961-4f31-96f9-d2267ec97f3a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546812407Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_415ee469-d5a2-4856-93b1-6e5d1fb10bbb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546816697Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_pantry_9ec1ec86-e462-408e-bb05-5b443483f792"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546820927Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_4de32ba2-a881-47b5-acf6-8242e72b57bc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546825087Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_external_dns_870344ce-2728-444f-8d5a-e3771e15477e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546829647Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_c8663988-f19e-423f-a69e-a5c3de3a9f25"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546833547Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_788d2519-7437-4048-92bd-a1a85eea6416"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546839927Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_crucible_0aecbbe1-3a43-407e-bceb-6b3134c30e9f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546914929Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_external_dns_71878588-3bc5-4be9-bddb-893f09285184"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.546921829Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_nexus_bc5aeb1b-883b-4e3c-84e4-bb4b98458327"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.554260485Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.561949808Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.570181524Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f36efe4f-a46b-459b-afc6-577a001d0331","remote_addr":"[fd00:1122:3344:101::1]:46418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34903721,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.585377697Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1415"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.60011688Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:790"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.605133393Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.639648951Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1017"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.643829926Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.76412182Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.880455024Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:40.968775277Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:03:41.017607099Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:03:41.084697442Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:41.138310972Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:41.431867225Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:41.766357098Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a)","v":0,"name":"SledAgent","level":50,"time":"2025-01-09T01:03:42.020530516Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:42.194424549Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:42.500882323Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/122d04f2-96f1-44ac-9122-b16b480179a5/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:43.090370764Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b3027db-248c-4f84-8bf8-26ab0cf033c0\", \"content-length\": \"113\", \"date\": \"Thu, 09 Jan 2025 01:03:43 GMT\"}; value: Error { error_code: Some(\"Forbidden\"), message: \"Forbidden\", request_id: \"8b3027db-248c-4f84-8bf8-26ab0cf033c0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:43.764135815Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"080151f9-4eab-4ace-813a-5dc1234785a8\", \"content-length\": \"173\", \"date\": \"Thu, 09 Jan 2025 01:03:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"122d04f2-96f1-44ac-9122-b16b480179a5\\\"\", request_id: \"080151f9-4eab-4ace-813a-5dc1234785a8\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:44.421698481Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ea714ff-2c70-40f5-a5ec-29b8fcba7b1c\", \"content-length\": \"173\", \"date\": \"Thu, 09 Jan 2025 01:03:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"122d04f2-96f1-44ac-9122-b16b480179a5\\\"\", request_id: \"1ea714ff-2c70-40f5-a5ec-29b8fcba7b1c\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:45.239068582Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:45.27357033Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da58be18-dcbd-4d60-bd01-6c1ca76e74db\", \"content-length\": \"173\", \"date\": \"Thu, 09 Jan 2025 01:03:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"122d04f2-96f1-44ac-9122-b16b480179a5\\\"\", request_id: \"da58be18-dcbd-4d60-bd01-6c1ca76e74db\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:46.175742995Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:46.97558742Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:46.980562993Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6fd0240a-0836-432c-b30c-4e2eaa6949c4","remote_addr":"[fd00:1122:3344:101::b]:61347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4603,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.011526701Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.01280399Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f2fbdfe5-6d96-4a12-b3ca-c5ea56a3b616","remote_addr":"[fd00:1122:3344:101::a]:53827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.124788807Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42324"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.134750081Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.135300584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.135628741Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.136208804Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34611"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.136352168Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.139706963Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.146327322Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.146382444Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7f4222f3-6419-413c-b3e1-d50eeaf1bce1","remote_addr":"[fd00:1122:3344:101::a]:53827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1629,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.156970873Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.156997813Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.157071725Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.162317303Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2dc4b293-e9bf-4483-bfd5-655a158f51c2","remote_addr":"[fd00:1122:3344:101::c]:60256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.182661782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.182774415Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.183127723Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.193901706Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.194335116Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3bf3cc16-73bf-477c-b3a7-82bdcb10847a","remote_addr":"[fd00:1122:3344:101::b]:61347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6131,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.198479949Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.19850472Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.215391671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.215435452Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.23928866Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41703"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.244009186Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60998"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.244708992Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.244734893Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.244876666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.24549647Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.249325097Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.249792507Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.256237362Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"bfded395-07cf-4e1e-b52c-7d6ec5f50f82","remote_addr":"[fd00:1122:3344:101::b]:41703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":11860,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.26324742Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.263276811Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"36fc8ca7-f87a-461b-bfc3-575206035696","remote_addr":"[fd00:1122:3344:101::c]:60256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.273063712Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.273090083Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.294268151Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.294326252Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.312233206Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"59a95668-027b-4973-a181-a5d0e9dff252","remote_addr":"[fd00:1122:3344:101::a]:53827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9011,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.316072972Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.316089173Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.324609635Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.32525363Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8541a169-2fbf-4657-af27-bd2147188c4a","remote_addr":"[fd00:1122:3344:101::a]:60227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":290,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.343516382Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.343642885Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.361542788Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:486","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 6 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 6 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.361654701Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"eada9ee4-5c35-43c0-b79d-c27b7dfd91c8","remote_addr":"[fd00:1122:3344:101::b]:61347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":396,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.36780379Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.36782933Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.379831161Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:486","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 6 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 6 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.380028385Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"40bd860f-ef3f-4ab5-9f68-c4237b72ae3d","remote_addr":"[fd00:1122:3344:101::c]:60256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":477,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.38642525Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.38643857Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.404717802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.404766683Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.423758842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.423801073Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.427378394Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.431573268Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b7d3a103-c803-493f-964c-d6b3d021b00c","remote_addr":"[fd00:1122:3344:101::c]:38124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":299,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.43962635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.43964862Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.454462405Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.454538566Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.468865959Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.468923881Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.484471312Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.484554404Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.498487138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.498541259Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.513472756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.513535927Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.528944865Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.528990316Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.543057724Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.543118085Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.556899466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.556977547Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.570499193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.570546494Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.584416177Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.584457818Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.600031809Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.60009036Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.604259254Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"f7d1dec6-f6ea-4de9-a687-64951173406a","remote_addr":"[fd00:1122:3344:101::a]:42324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":479347,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.613731578Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.613744728Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.628293507Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"18171c90-791f-4529-b2a3-975461561eb0","remote_addr":"[fd00:1122:3344:101::b]:34611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":491948,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.628897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.628921461Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.654230062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.654285613Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:47.668021193Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"da0d3991-a531-43e2-ae1a-61542b2618cb","remote_addr":"[fd00:1122:3344:101::c]:60998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":423476,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.706559554Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39455"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.706687287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.706694337Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.706736788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.707027435Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.719250751Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.719293792Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.749328859Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.74937192Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.773377962Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.773435723Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.801305392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.801345573Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.828337132Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.828379543Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.852543298Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.852583909Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.876057388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.87610314Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.899298883Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.899343894Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.921801571Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.921847982Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.935453739Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"68a11b18-b280-412b-8f09-c5c1d24f207e","remote_addr":"[fd00:1122:3344:101::c]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":228796,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.981306473Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60365"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.985284453Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.985891357Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.98603484Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.986072341Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.986089601Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.986103542Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.986153803Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.986410038Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4185ec90-995b-4661-9aed-1f1ff8beafc4","remote_addr":"[fd00:1122:3344:101::a]:60365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4993,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.988654919Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1089"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.993935878Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:48.994263435Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.031541597Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33125"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.031931825Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.031982557Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032003957Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032015957Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032022938Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032032808Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032038758Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.032059428Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"607c069c-8951-4305-a5fc-32f7418d1e56","remote_addr":"[fd00:1122:3344:101::b]:33125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":447,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.192277253Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57890"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.192463507Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193202754Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193231305Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193242635Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193252545Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193266576Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193271485Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.193290026Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ae331275-727c-4ed7-946b-5aa7abea76e9","remote_addr":"[fd00:1122:3344:101::c]:57890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":950,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.24134784Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33060"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.241417522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.241425392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.241435352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.245972105Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.258902346Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.258981808Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.284217308Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.284283269Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.30605249Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.306108201Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.329284194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.329337755Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.349611573Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.349662614Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.369770718Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.369818819Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.39073035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.390834973Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.413975195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.414028956Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.433964696Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.434005387Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.445514767Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"152ed241-549f-47f8-a5a0-fa2aceaad6c8","remote_addr":"[fd00:1122:3344:101::a]:33060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":204113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713161945Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33599"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713433001Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713494562Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713530433Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713553294Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713572374Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713589635Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713615515Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:49.713644956Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"29d44f8d-129c-4af5-9b0a-9522d9e4f38c","remote_addr":"[fd00:1122:3344:101::a]:33599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":384,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:58.875940723Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:58.876080987Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"86df336c-9b44-49aa-a9ac-b76098f90bfb","remote_addr":"[fd00:1122:3344:101::a]:50195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:58.977020794Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:58.977047415Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"aecb1c1f-6633-4111-b8c6-1775ab1818ac","remote_addr":"[fd00:1122:3344:101::a]:50195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:03:58.986841446Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cd3b0ab0-ae49-4d6e-bc9b-ab2286f576fd","remote_addr":"[fd00:1122:3344:101::a]:50195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:02.159865305Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:02.162030504Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"ccbe4fe6-2d20-47ae-a602-a43eefa0fcb8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45971","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:16.81004062Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:16.810145412Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d703d624-e1bb-4278-acae-b60f57512a1c","remote_addr":"[fd00:1122:3344:101::a]:37624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:16.84373144Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:16.843768191Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d6671768-1103-4499-86ff-d2930f25291d","remote_addr":"[fd00:1122:3344:101::a]:37624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:16.857363958Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3acd5f03-ee06-4a83-a23b-15700eee09de","remote_addr":"[fd00:1122:3344:101::a]:37624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.02278738Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.023137148Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"21b54392-958b-4efc-9fed-40f9ea23e628","remote_addr":"[fd00:1122:3344:101::b]:54324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.023168049Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.023260911Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6765e091-9bdc-4d67-9aa2-f8de28047385","remote_addr":"[fd00:1122:3344:101::b]:59930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.073401652Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.073628447Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c33be74d-5f90-426b-8ef3-3d8f85736741","remote_addr":"[fd00:1122:3344:101::a]:38500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.090314304Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.090354054Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d4ff2fa4-abce-4038-a6c0-a35bcd0d5694","remote_addr":"[fd00:1122:3344:101::b]:59930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.105515467Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4f3a7176-7a45-4464-9cea-8c3580d82ec4","remote_addr":"[fd00:1122:3344:101::b]:59930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":297,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.216715725Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.217571895Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.306862789Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377033152Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d6310e38-0525-4e5b-b559-899330281276","remote_addr":"[fd00:1122:3344:101::a]:43177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377287268Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33479"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377343609Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38436"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.37739085Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377680947Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ac129bd9-a430-4295-9d8a-8d6af8c930a2","remote_addr":"[fd00:1122:3344:101::c]:43217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377868501Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.377967483Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"3c3f9cac-3a0e-4398-be98-e909997abb99","remote_addr":"[fd00:1122:3344:101::c]:38436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":379,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.389682158Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"dfa53366-b7e5-4556-ae51-427e4ecc0d02","remote_addr":"[fd00:1122:3344:101::a]:43177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":12325,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.43720534Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.442586061Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.442685823Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d6104a79-a917-47aa-af04-b3f7584dd67b","remote_addr":"[fd00:1122:3344:101::c]:43217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":706,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.444481374Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33919"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.447742577Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.4496207Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.450745275Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.452356152Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.453254272Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"PUT","req_id":"65f213db-f682-421f-9d2b-e740cafe5954","remote_addr":"[fd00:1122:3344:101::a]:43177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":11368,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.460761861Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"aa897c34-2a45-4252-ac91-3568ef3b9414","remote_addr":"[fd00:1122:3344:101::c]:43217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":841,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.510735279Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.51079519Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e468340c-239a-4304-aba9-db8643051248","remote_addr":"[fd00:1122:3344:101::b]:33479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.51655733Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"460ec904-bf32-43db-8db7-06224477a305","remote_addr":"[fd00:1122:3344:101::b]:33479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1436,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.554819964Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.614656194Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dfdc5c52-5d12-4ee0-9763-e84c5b0b80c2","remote_addr":"[fd00:1122:3344:101::c]:33919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:17.615108734Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"42745c16-1b71-489c-bc02-c84a6e63ac4d","remote_addr":"[fd00:1122:3344:101::c]:33919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:32.196096616Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45165"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:32.196218009Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d73b8908-c9e6-4be9-87cb-8eee4d2b61bd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45165","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:41.028583182Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.944715178Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.944857682Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"55fd6e06-051d-440e-b354-b1f10bb51d0b","remote_addr":"[fd00:1122:3344:101::a]:56703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.945414904Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.945566248Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c79d31f2-141d-4a6d-9322-555f142ae74f","remote_addr":"[fd00:1122:3344:101::b]:37830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.979218987Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:46.979251748Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e3086bd9-264e-49f4-8f04-3092a93a49c6","remote_addr":"[fd00:1122:3344:101::a]:56703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.000833385Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3ec8094d-2616-4874-a282-0be5434bd96b","remote_addr":"[fd00:1122:3344:101::a]:56703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.025542192Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.025566523Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"10eabad0-790b-4f84-be15-ec850eec0399","remote_addr":"[fd00:1122:3344:101::b]:37830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.137812275Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41674"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.138048831Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4bd487d8-5e78-43c8-afcb-a2ba804f2673","remote_addr":"[fd00:1122:3344:101::b]:41674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":193,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.148809723Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"394214ea-6cec-40c7-b8cd-ea4631c737b9","remote_addr":"[fd00:1122:3344:101::b]:37830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.176344875Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35569"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.176449857Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.222000454Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.281040697Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5cf38730-596f-488e-a067-73358acdee6e","remote_addr":"[fd00:1122:3344:101::a]:35569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104641,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.281137539Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40707"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.281163589Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46345"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.2812027Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40290"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.281256631Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58572"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.281461756Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.285539448Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f30ef427-7026-424d-998d-471834d63570","remote_addr":"[fd00:1122:3344:101::a]:40707","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4084,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.285679091Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c00ed49e-cdb0-43f6-a90e-015cb13925bb","remote_addr":"[fd00:1122:3344:101::a]:35569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4018,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.285713892Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2c92beb9-2b88-41b0-9623-b1175513dc08","remote_addr":"[fd00:1122:3344:101::c]:46345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.285807894Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d7a4e851-ba4f-427d-a392-53626e377d67","remote_addr":"[fd00:1122:3344:101::c]:58572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4297,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.302851879Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.30290746Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0e79fd71-908b-4708-b3aa-4872813b8b9d","remote_addr":"[fd00:1122:3344:101::c]:46345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.313281934Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0e19df96-2fd0-419e-8527-a75b27873f17","remote_addr":"[fd00:1122:3344:101::c]:46345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":380,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.318559543Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.374746191Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"daf66b6d-8f19-4809-8907-716bc35c493a","remote_addr":"[fd00:1122:3344:101::b]:40290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93402,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.375124949Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b5d912c3-d87a-42e2-a402-5051dbae5dc4","remote_addr":"[fd00:1122:3344:101::b]:40290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.442879598Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36724"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.443008291Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.479370081Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.535569579Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e2b0f133-9db3-449f-80cd-07a81b7c75c1","remote_addr":"[fd00:1122:3344:101::c]:36724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92618,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:04:47.535814374Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"13465853-cdd5-4f79-8cd8-7634d8c3a680","remote_addr":"[fd00:1122:3344:101::c]:36724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:02.230790994Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:02.230920997Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"03299aea-14f3-40bd-a9ed-e6d5481ceda0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50639","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:14.941886808Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:63574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:15.033136657Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"8de605e4-2c6f-42e3-90ac-6bdc87067c3f","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87223,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.900926357Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.901072371Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"bd567ba6-cd74-4a5e-b67e-44c743d7d907","remote_addr":"[fd00:1122:3344:101::a]:52116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.90193993Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.902249817Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f964092f-f84f-4192-93b7-85f18f3b3d94","remote_addr":"[fd00:1122:3344:101::b]:34399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.913329507Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57410"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.913558562Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b6f4381b-42d7-4010-b49e-695bb2c74895","remote_addr":"[fd00:1122:3344:101::b]:57410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.966569898Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.966602549Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b8413e67-fc21-4eaa-a8be-26f688d37bfa","remote_addr":"[fd00:1122:3344:101::b]:34399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.972578224Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.972603395Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"52408035-8c42-455f-8e45-795999e901e9","remote_addr":"[fd00:1122:3344:101::a]:52116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.985431564Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ceadc95a-3e36-4c7c-ba54-bda315a478e1","remote_addr":"[fd00:1122:3344:101::b]:34399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":261,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.988921383Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"acbb9cb1-1b68-4eb8-bf09-8d51a1f78cc4","remote_addr":"[fd00:1122:3344:101::a]:52116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.998685773Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51085"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:16.998787815Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.049925719Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.122056366Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9de0de6a-e0cd-4298-b0d9-204fe5f8ea26","remote_addr":"[fd00:1122:3344:101::a]:51085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123301,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.12222664Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49949"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.122444175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.127260574Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1ffec24a-4eeb-4417-8394-2853c7677d74","remote_addr":"[fd00:1122:3344:101::a]:51085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4775,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.145391253Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.14568464Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7b7848b4-0a42-432b-935e-97f0e40439be","remote_addr":"[fd00:1122:3344:101::a]:49450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.145745051Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50857"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.145877184Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"63b83492-f804-4168-a6e6-951466ac6321","remote_addr":"[fd00:1122:3344:101::c]:50857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.165063827Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.169151999Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.170590292Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4eb0a9ee-687d-4132-9eea-019a24fef28a","remote_addr":"[fd00:1122:3344:101::c]:37206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":317,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.196693201Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.196766312Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a88bb974-27b0-4eb1-b331-7ad43f948ae1","remote_addr":"[fd00:1122:3344:101::c]:50857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":667,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.209389937Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"665f70ab-2dc7-4901-ad7b-ef9c2f5c7f47","remote_addr":"[fd00:1122:3344:101::c]:50857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4294,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.228715973Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5118953e-26ad-4751-a1dc-393234129ca8","remote_addr":"[fd00:1122:3344:101::b]:49949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106362,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.229086381Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"86c3f393-2e66-459f-b54c-d645552db792","remote_addr":"[fd00:1122:3344:101::b]:49949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.288991373Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39064"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.289113476Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.325355133Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.385227133Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2d2980a9-7d41-4b26-9715-9f77e87d0ab1","remote_addr":"[fd00:1122:3344:101::c]:39064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96163,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:17.385628782Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"65beb2ec-00ae-4314-b0a2-426396cad28a","remote_addr":"[fd00:1122:3344:101::c]:39064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:32.274237563Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35562"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:32.274367496Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a4aa49ed-df70-437d-9d8a-cfe7b943a50f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35562","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:44.94908854Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"a7cdafcd-ea44-4a67-a674-27eebf4c90d3","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.006532709Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.006753344Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"74f28800-1f7b-4e98-9e69-841a0ff799ba","remote_addr":"[fd00:1122:3344:101::b]:64017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.012863402Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.012992614Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0b11d2ee-156f-4558-8bcb-821ba543e84e","remote_addr":"[fd00:1122:3344:101::a]:50862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.079326841Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.079357212Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8f074265-9689-4afd-907d-df3d6fc9e907","remote_addr":"[fd00:1122:3344:101::a]:50862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.088022467Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b05e19db-6914-4441-bbbc-6aaf26e068f5","remote_addr":"[fd00:1122:3344:101::a]:50862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":196,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.158226251Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.158328544Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8b36a9d2-3ce8-4fb4-b561-e43bdf342356","remote_addr":"[fd00:1122:3344:101::c]:52205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.17456985Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48275"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.174734624Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7abbf3f2-b7ac-47b3-b374-a5a432f375ef","remote_addr":"[fd00:1122:3344:101::b]:48275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.178927198Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.179177894Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"dbf9e2ce-aa28-408c-9ab2-41644a97d491","remote_addr":"[fd00:1122:3344:101::a]:36991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.184530985Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.184553445Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"dd9dd6ba-2d2b-476e-955e-8a11f266ed06","remote_addr":"[fd00:1122:3344:101::c]:52205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.195352689Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1e034909-87fe-4a8d-b786-1153d87c2fac","remote_addr":"[fd00:1122:3344:101::c]:52205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.203201656Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.203221486Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2e18a1c6-004a-46fb-b4d3-5425f40d14c4","remote_addr":"[fd00:1122:3344:101::b]:48275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.213410336Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4f704f35-7da9-42c4-8d31-1c67df61b65c","remote_addr":"[fd00:1122:3344:101::b]:48275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.312577234Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40405"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.312688636Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.350801015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.403560626Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"822fbed0-5142-4533-aae6-97547dbf68aa","remote_addr":"[fd00:1122:3344:101::a]:40405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90917,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.403886923Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"59276614-52cd-4c14-bdc0-34bc53edc42f","remote_addr":"[fd00:1122:3344:101::a]:40405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.423105657Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.423350142Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"05485b7f-6524-40f7-b5db-0a0bbab0ad66","remote_addr":"[fd00:1122:3344:101::c]:61654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.449186375Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34728"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.449292028Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.488571484Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.546397338Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"835e4270-39cc-40d9-af75-36d653e6616e","remote_addr":"[fd00:1122:3344:101::c]:34728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97158,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.546708466Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cd2bc2ee-83e8-496c-bfaa-d5bcbca618e7","remote_addr":"[fd00:1122:3344:101::c]:34728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.589831328Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50921"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.590026223Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.630016325Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.692901174Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d4fd65a9-4a20-422f-b8fb-6a516a260a26","remote_addr":"[fd00:1122:3344:101::b]:50921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102940,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:05:47.693245942Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3c412a1a-417e-4268-9c38-1e8191bbb89e","remote_addr":"[fd00:1122:3344:101::b]:50921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:02.306198375Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:02.306321748Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0460d73e-7153-48d4-809a-512532520931","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37852","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:14.948003505Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"41488ae4-854e-4d1a-a7d5-e147233e0641","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7434,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.86963356Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.869788023Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"136b8b68-647e-40b5-9a3c-4291a54ad9fa","remote_addr":"[fd00:1122:3344:101::b]:34049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.878254894Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59511"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.878361437Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ec392041-3904-49b6-bb7a-137e4ae0a3ee","remote_addr":"[fd00:1122:3344:101::a]:59511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.887170625Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.887224287Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ae3b64e7-00e0-4f50-9a62-e5a46e403347","remote_addr":"[fd00:1122:3344:101::b]:34049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.898092882Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.898271376Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"25d41882-5d95-470a-b3ac-938de2d394a9","remote_addr":"[fd00:1122:3344:101::b]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.901712773Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"64c5003c-22d9-441b-8037-f2515907e737","remote_addr":"[fd00:1122:3344:101::b]:34049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.937220975Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:16.937438739Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"1ef04952-a61e-432f-90ba-6fcc4272bf0d","remote_addr":"[fd00:1122:3344:101::a]:44340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.038105561Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.038147762Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e7d95e35-4dde-4ce5-9d23-16a204a5e84a","remote_addr":"[fd00:1122:3344:101::a]:59511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.046443699Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"bc3916fb-8ea1-4246-a62f-c70b5131cf30","remote_addr":"[fd00:1122:3344:101::a]:59511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":223,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.092212362Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.092288993Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c65bd728-f262-43da-81ff-6ba6c2e1bf73","remote_addr":"[fd00:1122:3344:101::c]:53105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.095595608Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56560"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.09569503Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.138459095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.19896786Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0543f770-0053-4a49-9c8a-996104926cd3","remote_addr":"[fd00:1122:3344:101::a]:56560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103310,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199132644Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199241176Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39412"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199300698Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33125"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199512062Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199587514Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2d7a7e83-39f6-4faf-b0ca-9bb43928d71f","remote_addr":"[fd00:1122:3344:101::c]:53105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":291,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199620475Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"88a0333b-7cc0-4e8f-b433-1b7dd4994dbc","remote_addr":"[fd00:1122:3344:101::a]:56560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":314,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199642955Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4297a103-0405-4ef7-b8ab-55e81553295e","remote_addr":"[fd00:1122:3344:101::c]:33125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":232,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.199670976Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.210406558Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"44172c20-0f0b-4a33-9522-518d6b0683ea","remote_addr":"[fd00:1122:3344:101::c]:53105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":537,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.236163049Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.290864773Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.290905044Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e0ff8d83-4f89-49a6-955a-16a32865f2e9","remote_addr":"[fd00:1122:3344:101::b]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91284,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.294882414Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"853f73c7-6fc0-4775-9a2b-c68b7222f548","remote_addr":"[fd00:1122:3344:101::b]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":734,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.326801834Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.384081106Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dbf82587-b760-4ee6-824c-cd121fb7d46c","remote_addr":"[fd00:1122:3344:101::c]:36355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184465,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:17.384376963Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d5b37191-e438-4551-86a8-f2457c5574ad","remote_addr":"[fd00:1122:3344:101::c]:36355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:32.338610405Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36725"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:32.338800439Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"cb9b7c26-0fbb-4c99-ba4b-d429c0eb1aca","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36725","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:44.948629676Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"064dd80b-caa1-45e9-b467-8cf428ad40ca","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7650,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:46.991154488Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:46.991304022Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"bfb14b9b-c447-404b-8272-5414f2811880","remote_addr":"[fd00:1122:3344:101::a]:59422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.057876444Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.057969136Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"706315cb-b424-45ec-8fb2-44afcc55045c","remote_addr":"[fd00:1122:3344:101::b]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.076651117Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.076887483Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0e92cfb8-e8d6-4529-ae11-084dbf57aee4","remote_addr":"[fd00:1122:3344:101::a]:61414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.112161738Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.112183419Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a65c7663-e2b2-4f3b-91e3-9cc2a9a783b5","remote_addr":"[fd00:1122:3344:101::b]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.162560796Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.162587156Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f600b8f3-713c-4978-8083-95039db776f5","remote_addr":"[fd00:1122:3344:101::a]:59422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.172106151Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8584f319-7206-40ee-b439-229563d25015","remote_addr":"[fd00:1122:3344:101::a]:59422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":272,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.203793836Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.203918179Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"33bf26c5-291e-4a1f-96fe-4826ff8bb5d6","remote_addr":"[fd00:1122:3344:101::b]:49666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.264761521Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37949"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.264856593Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.305738186Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.366768613Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"578ad0d5-5f73-471e-a948-4f546525b8ca","remote_addr":"[fd00:1122:3344:101::a]:37949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101943,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.366976657Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.367341706Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"530b11c4-c526-4c8f-b40d-ed9c48e1cbdb","remote_addr":"[fd00:1122:3344:101::a]:37949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":269,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.367387537Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"65526376-26a5-4d38-863e-fb4c0e2da348","remote_addr":"[fd00:1122:3344:101::b]:42754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":437,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.380874381Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.386221762Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.386407226Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6d5def32-ff2a-4086-a392-d3ed29c7a815","remote_addr":"[fd00:1122:3344:101::c]:38805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.427051053Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.502028324Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"51df91cb-d85b-4d2e-a0f6-93206cc0ec1e","remote_addr":"[fd00:1122:3344:101::b]:60435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133601,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.502426113Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9e4f3907-a95b-458f-b745-a62c33f1b7d2","remote_addr":"[fd00:1122:3344:101::b]:60435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.519096089Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.51913301Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f665667d-9fbb-413e-b8a3-3bb4b2e68086","remote_addr":"[fd00:1122:3344:101::c]:38805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.586302696Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39896"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.586429419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.629629333Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.690431965Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6612da3e-d87a-419f-ac12-7cdf33b93e88","remote_addr":"[fd00:1122:3344:101::c]:39896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104055,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.690746812Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.690957977Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6b35618b-ae63-40fd-8d1b-d6f30a8d8870","remote_addr":"[fd00:1122:3344:101::c]:39896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.691048379Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"34ae54ac-fdc6-4e23-98b0-ed5fa02fb841","remote_addr":"[fd00:1122:3344:101::c]:38805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":451,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:06:47.691152471Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"96e3ae4a-1336-48f2-98bf-a878a787f70d","remote_addr":"[fd00:1122:3344:101::c]:47350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:02.371101744Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39890"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:02.371237797Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"fd6c8633-2f34-4386-a0b4-e439ee0dd6a7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39890","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:14.950066795Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"4fbdf34b-91d3-4373-8d1f-3af781b383ef","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8194,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.781441064Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53148"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.781557287Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7740c60e-ec2b-477a-be81-d85a34ff4105","remote_addr":"[fd00:1122:3344:101::b]:53148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.809310993Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.809344163Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b6dbc10f-cc14-473e-8ddc-3ff2515cd346","remote_addr":"[fd00:1122:3344:101::b]:53148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.822249045Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"44873d63-abbd-47e3-8d2c-cc0b39f0c3de","remote_addr":"[fd00:1122:3344:101::b]:53148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.839467443Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44083"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.839557375Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"956efb1b-534f-4aec-8c31-9f8b9054735a","remote_addr":"[fd00:1122:3344:101::a]:44083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.894871223Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.894907364Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b5afed7e-5bd9-49a6-a99a-1eeb9b72fd35","remote_addr":"[fd00:1122:3344:101::a]:44083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.906945056Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50373"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.907052848Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:16.966123964Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.045765308Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5d8f4afa-3054-419c-a095-d6e2cab2d7e4","remote_addr":"[fd00:1122:3344:101::a]:50373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138751,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046046034Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35666"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046167857Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44381"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046245068Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046428713Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c3c97c53-8e01-4187-8740-f4234cfe810f","remote_addr":"[fd00:1122:3344:101::a]:50373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":341,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046626227Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2489b8f1-2177-4506-b125-19f4fb9a1c86","remote_addr":"[fd00:1122:3344:101::a]:44083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":630,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046647737Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.046699519Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"46f90fde-ffa4-4407-b251-ffe1fccc34a9","remote_addr":"[fd00:1122:3344:101::b]:35666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.0467479Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e1857e78-8309-437a-b30d-febc20049873","remote_addr":"[fd00:1122:3344:101::a]:44381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":295,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.086536288Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.14070093Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"97c6a885-5634-4ebf-9833-5a0094decc7f","remote_addr":"[fd00:1122:3344:101::b]:58320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94257,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.141119579Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ecd716a5-0a04-46cf-9a46-d38a43be7c2c","remote_addr":"[fd00:1122:3344:101::b]:58320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.197185414Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47811"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.197287676Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1a1a61e1-b78c-4b0e-8db0-862c1edbaa81","remote_addr":"[fd00:1122:3344:101::c]:47811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.217662646Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.217694717Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"76d4b24a-928b-4d25-a5e1-b14ed21932d4","remote_addr":"[fd00:1122:3344:101::c]:47811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.255982801Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38364"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.256158525Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f19c3bdf-285a-4501-9047-3bd14b128981","remote_addr":"[fd00:1122:3344:101::c]:38364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.264903232Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7b8e2601-9483-4842-b70e-84855643bee8","remote_addr":"[fd00:1122:3344:101::c]:47811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.317210662Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35445"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.317309515Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.35743934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.413599127Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"47f241c6-c9a5-42b7-ad67-77a4f3c900a2","remote_addr":"[fd00:1122:3344:101::c]:35445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96332,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:17.413858793Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"be3cfbd4-c390-4491-8235-2c1068a11949","remote_addr":"[fd00:1122:3344:101::c]:35445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:32.405074224Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62290"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:32.405225307Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d195b00c-0b33-4b06-9e28-9f9d9c9b7b2b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62290","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:44.948865173Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"c5950a06-4dbd-4e41-8fed-ee83a7f5627a","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7761,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:46.967478057Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:46.96761556Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e81629f4-a284-4641-a8fb-d70f787f8bb6","remote_addr":"[fd00:1122:3344:101::a]:64293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.057693573Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.057721913Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"03cd1fe5-ed6b-4b40-a3b1-fa2b6cc7c09d","remote_addr":"[fd00:1122:3344:101::a]:64293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.068482346Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3acc4de4-b402-4485-a84c-afe1c0e30945","remote_addr":"[fd00:1122:3344:101::a]:64293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.117163074Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.117364529Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"29f2ec0e-e778-4755-8913-4f8be3c1414e","remote_addr":"[fd00:1122:3344:101::b]:60072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.122569176Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58008"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.122658648Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"dd5d5a9e-7655-4f5a-8281-5235bcbf2bb7","remote_addr":"[fd00:1122:3344:101::b]:58008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.138406074Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37623"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.138534887Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.181390343Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259153468Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3a60691d-c44b-428e-b23f-8bf60aba471b","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120652,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259338972Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42803"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259463185Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259720991Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6e94ac77-ce1e-4fe4-b961-acf2d39977e0","remote_addr":"[fd00:1122:3344:101::c]:42803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259770392Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259808803Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"23baeba7-7b85-4b22-bc82-f68ac3199c79","remote_addr":"[fd00:1122:3344:101::b]:58008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":273,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259840724Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7cf1d9ce-50f2-46e9-bb81-923f32811ac8","remote_addr":"[fd00:1122:3344:101::a]:37623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":344,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.259906945Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8f59a93f-066b-4416-af18-ec006cc9f7ef","remote_addr":"[fd00:1122:3344:101::a]:36590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.274588906Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b4459cec-9e04-43ff-89ba-75ed5e1d5ac8","remote_addr":"[fd00:1122:3344:101::b]:58008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":370,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.304530212Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.304550972Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7f4805b1-4bfc-4518-b6d0-510bf9fbbddf","remote_addr":"[fd00:1122:3344:101::c]:42803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.309808891Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33972"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.309916473Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.353587379Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.410925642Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e9084cd6-1ead-4bec-a119-04abdc739a64","remote_addr":"[fd00:1122:3344:101::b]:33972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101050,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.411376732Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0a4fe25d-bbbc-4d8a-8ffc-a57bb574a182","remote_addr":"[fd00:1122:3344:101::c]:42803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":293,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.411543596Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"70db39d9-3814-45cd-bb54-d48853a596cd","remote_addr":"[fd00:1122:3344:101::b]:33972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.416773624Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.417008799Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"668666cd-aa47-40ed-80cd-768609929ec9","remote_addr":"[fd00:1122:3344:101::c]:36248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.520103505Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63450"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.520228768Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.560966998Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.622469105Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3a67f57e-bc16-44da-b842-503505325706","remote_addr":"[fd00:1122:3344:101::c]:63450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102308,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:07:47.622721001Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"430eed29-5ef6-4b18-bbc9-85fdfe4693b0","remote_addr":"[fd00:1122:3344:101::c]:63450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:02.436953295Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:02.43716426Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d3545c57-0780-439e-aa5c-aae3da805f10","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53521","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:14.948811186Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"fac5d498-afed-4173-8a97-dff910386984","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7800,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.878747548Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56043"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.878854351Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"827dd24b-6fa8-427c-8f4c-d20d452a61c0","remote_addr":"[fd00:1122:3344:101::b]:56043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.938585918Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.938618239Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"299b86c7-4584-43e4-8775-089ff1fe84c6","remote_addr":"[fd00:1122:3344:101::b]:56043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.94930666Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"22788b89-392c-4e81-84fe-1a7ff92daeb5","remote_addr":"[fd00:1122:3344:101::b]:56043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":261,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.953489705Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41870"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:16.953664449Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"984d09cd-2554-448b-8496-9a3eaf8cbdd6","remote_addr":"[fd00:1122:3344:101::b]:41870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.024606169Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.024797843Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f5e0753b-91b0-4668-971f-ed5ea5b519e8","remote_addr":"[fd00:1122:3344:101::a]:36080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.030987073Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.031047715Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6bbea5aa-1ef2-46c1-857d-5404c1026dad","remote_addr":"[fd00:1122:3344:101::a]:54477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.076974961Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.077007132Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"49ee4862-1ce8-4d32-ac3b-6c32c4c02e18","remote_addr":"[fd00:1122:3344:101::a]:54477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.08582836Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37282"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.085930943Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.129080946Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.188837895Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fe6af25e-9390-4a29-9db8-ce7377443010","remote_addr":"[fd00:1122:3344:101::b]:37282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102939,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.189129041Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49655"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.189453098Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0240acf2-6f0b-4998-9c06-50f49f54f6e4","remote_addr":"[fd00:1122:3344:101::b]:37282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":309,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.18952637Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5709b2d7-449d-431d-8f09-5f008ba9b77e","remote_addr":"[fd00:1122:3344:101::a]:54477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":488,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.189632082Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.214203977Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.214427452Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ca6c1a32-f86e-4f7b-982a-a1ad3812c47c","remote_addr":"[fd00:1122:3344:101::c]:35574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.229110893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.268995143Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.269155817Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"07124e44-2ac0-4499-a9bf-6ed83d6d6f44","remote_addr":"[fd00:1122:3344:101::c]:48491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.288075454Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6c6d57d1-5dcb-4dbd-a297-0d6427f5bd1f","remote_addr":"[fd00:1122:3344:101::a]:49655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98596,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.288424911Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b50695c0-d572-4c17-acd2-49d7bad3d189","remote_addr":"[fd00:1122:3344:101::a]:49655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.312800632Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.312841842Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6e8c63af-d249-4a52-a225-255f0bae6276","remote_addr":"[fd00:1122:3344:101::c]:48491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.319654606Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38006"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.319781619Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.359753931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.417012993Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"65304c86-d535-4ad6-9d54-18b9ff0e7297","remote_addr":"[fd00:1122:3344:101::c]:38006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97303,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.417453093Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e3c0c69d-a78a-4626-89d9-10da9d770f92","remote_addr":"[fd00:1122:3344:101::c]:48491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":288,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:17.417597206Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fa08e28a-4e86-4314-be4c-76c2de3315a6","remote_addr":"[fd00:1122:3344:101::c]:38006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:32.470537505Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:32.470694009Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"aa8889c9-9de4-40aa-9929-45061de0758f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55714","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:44.949478341Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"f7330826-b057-42e8-9ff2-8be12433b46f","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8473,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:46.892864367Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:46.8929937Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a26f7fa0-05ae-468d-a63c-e6e59eb66734","remote_addr":"[fd00:1122:3344:101::b]:55994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:46.992671339Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:46.992701729Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3bd1ce24-51b4-4b41-8d62-366fed58932f","remote_addr":"[fd00:1122:3344:101::b]:55994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":377,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.002774776Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.003043363Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"eb34dd6f-65c1-47da-862b-692d16d180b8","remote_addr":"[fd00:1122:3344:101::a]:48545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.034265897Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35934"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.034769258Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"78d912e1-9086-42da-83f1-48a81dfb90b2","remote_addr":"[fd00:1122:3344:101::a]:35934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.104538923Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.104570393Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a8648f3a-74f8-43b5-99ca-1438faf392fd","remote_addr":"[fd00:1122:3344:101::a]:35934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.1185942Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"29139208-4db0-4053-8346-5b30365da7d0","remote_addr":"[fd00:1122:3344:101::a]:35934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.146564121Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.146737915Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8bef434d-3524-4e73-bad3-c99eca659f39","remote_addr":"[fd00:1122:3344:101::b]:59011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.154264454Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cfffe95e-3294-4116-9fca-85d2accae75f","remote_addr":"[fd00:1122:3344:101::b]:55994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.211221559Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41404"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.211328012Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.257538904Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.330724896Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"28af4f2d-2a3f-48c6-ac82-824f3012367e","remote_addr":"[fd00:1122:3344:101::a]:41404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119445,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.330964941Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42863"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.331069374Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.337087229Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fd600be9-3140-4246-ac77-e7f10ceb65c6","remote_addr":"[fd00:1122:3344:101::a]:41404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5735,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.345041529Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.345224603Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"20745d8b-e5a5-40cf-a984-297f2f6dc7a5","remote_addr":"[fd00:1122:3344:101::c]:42863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5846,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.386586206Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.393819569Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.39386853Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d87a9a70-d5ac-43bb-8120-e20b37f8642a","remote_addr":"[fd00:1122:3344:101::c]:42863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.454334735Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"16189161-b327-4512-8bad-9d5d582760ef","remote_addr":"[fd00:1122:3344:101::b]:38987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117849,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.454733273Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"06cf339d-b9dc-4627-9551-660076ad3d19","remote_addr":"[fd00:1122:3344:101::b]:38987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.457779742Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57544"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458043988Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458096869Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.45813933Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458163141Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458191842Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458210102Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458263943Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.458290844Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f46fefd1-89e7-4a5f-93b4-824ab10843d7","remote_addr":"[fd00:1122:3344:101::b]:57544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":399,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514070072Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58834"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514224486Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514274277Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514300417Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514311278Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514340908Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514358899Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514377689Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.514392519Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7db08a98-caf3-48a4-af85-1ce87543052f","remote_addr":"[fd00:1122:3344:101::a]:58834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.559758333Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.559912526Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"dcc016df-06cb-4bc5-a2eb-98cf16963f4b","remote_addr":"[fd00:1122:3344:101::c]:48341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.569252097Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b6675d6a-ca2c-4f61-974f-557e17dd914e","remote_addr":"[fd00:1122:3344:101::c]:42863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.633854855Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54154"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.633969897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.676709142Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.737896442Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"378d68d1-e0c4-4420-9870-ce067f05e707","remote_addr":"[fd00:1122:3344:101::c]:54154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103963,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.738187269Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d2501187-2def-46f4-9837-597a0fad2298","remote_addr":"[fd00:1122:3344:101::c]:54154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.762724653Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52679"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.762975978Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763026769Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.76306927Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763090731Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763109521Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763119051Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763144962Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:08:47.763169332Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1666f35a-300e-4c03-8ab4-23bd8121ae4a","remote_addr":"[fd00:1122:3344:101::c]:52679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":366,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:02.505608955Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:02.505752048Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"547be653-7446-4bb1-9f5a-7bb05b2b4d89","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39787","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:14.951069305Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"cc81500a-300b-48eb-a987-57197dafc4d2","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7375,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.840912473Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.841010425Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8918f5a5-cfeb-4abe-89fc-14234f47ed5f","remote_addr":"[fd00:1122:3344:101::a]:36520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.846865787Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33623"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.846952169Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"35be03ad-6a28-4e6a-a979-4c1d7308c88f","remote_addr":"[fd00:1122:3344:101::b]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.889756075Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.889789296Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ddaa8949-93e3-4c2a-9beb-c1be6c0ec480","remote_addr":"[fd00:1122:3344:101::a]:36520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.900483947Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a47d6dd8-154c-4c49-903b-8f18f3f02999","remote_addr":"[fd00:1122:3344:101::a]:36520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.901525011Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.901559311Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4700b1f2-4783-42e8-863c-bb9e42b69bea","remote_addr":"[fd00:1122:3344:101::b]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.910617456Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"78432080-fce3-467e-925f-61ecca490b6a","remote_addr":"[fd00:1122:3344:101::b]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.951981619Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44161"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.952079781Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:16.997027955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.075867344Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dd5c280d-ccdb-465e-bc4b-7e99e1ba27ac","remote_addr":"[fd00:1122:3344:101::a]:44161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123830,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076079399Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35206"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.07614667Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41820"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076263673Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58190"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076339675Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.07657079Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"59371025-9bd9-4b42-8112-11c0110b59f1","remote_addr":"[fd00:1122:3344:101::a]:44161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":340,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076614821Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f9aa3ee8-625f-4890-9ca1-a38d8187e2e4","remote_addr":"[fd00:1122:3344:101::c]:46617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076712583Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.076799875Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8572f3f7-a29e-493f-a18e-13c33909710d","remote_addr":"[fd00:1122:3344:101::b]:41820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":291,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.089725227Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c87c50e7-d2eb-4485-81ff-3748b7a2ad75","remote_addr":"[fd00:1122:3344:101::a]:35206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7916,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.098209498Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.09829154Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"018eb210-3d5b-4bd5-ab47-501bdfb8828c","remote_addr":"[fd00:1122:3344:101::c]:46617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":655,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.106866644Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"083a5790-01d9-488d-b969-f93dfe3388b8","remote_addr":"[fd00:1122:3344:101::c]:46617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.121754149Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.185663811Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"42202293-a81f-46b2-b07c-95aec051ffa9","remote_addr":"[fd00:1122:3344:101::b]:58190","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109079,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.186011669Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"91eeec94-93dc-40cc-b334-79d9b7ebe314","remote_addr":"[fd00:1122:3344:101::b]:58190","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.198213774Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.199401181Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5ba2437c-9e5f-419e-8290-c94a0d20136b","remote_addr":"[fd00:1122:3344:101::c]:41925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.199451162Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56995"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.199574795Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.236009967Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.290572908Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"33ad5b7c-70d7-4d23-ba17-700a96b46b04","remote_addr":"[fd00:1122:3344:101::c]:56995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91046,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:17.290954617Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fe0451d3-770f-4f07-9d37-7f34269f9841","remote_addr":"[fd00:1122:3344:101::c]:56995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:26.361688928Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:26.362641729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:26.403545412Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999746s","next_cleanup":"Instant { tv_sec: 2009, tv_nsec: 863410641 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:32.547899438Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:32.548093513Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3640c25e-1c0e-4210-a7de-e5736b8faef1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37968","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:41.030911419Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:44.949390247Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"fcda6847-eccd-40e6-b4ce-88fe4156703e","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7967,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:46.885718204Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44674"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:46.885873988Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8c406c2a-0f60-42c6-b77a-b407ac72aa49","remote_addr":"[fd00:1122:3344:101::b]:44674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:46.987637724Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:46.987692075Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0d328c02-5f62-4e7d-84f5-015e119ebd4f","remote_addr":"[fd00:1122:3344:101::b]:44674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.022602673Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f4fe71e1-b15f-4149-9f68-8c250488fdd7","remote_addr":"[fd00:1122:3344:101::b]:44674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":405,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.043623857Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35829"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.043809591Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8be9ee45-bc34-4a82-a6b3-6c7a1800678c","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.066920063Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.066955283Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"40edeee8-9eba-4772-8e5a-5f7325a6f66d","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.077538392Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ebae3c15-0ca4-4f31-915c-519819304b1a","remote_addr":"[fd00:1122:3344:101::a]:35829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":314,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.169507537Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49698"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.16964747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.212746393Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276183324Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"90e6a402-e27b-4958-9a3c-ba91a0e877a4","remote_addr":"[fd00:1122:3344:101::b]:49698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106602,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276320157Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44344"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276358518Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47890"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276419839Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58318"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.27645697Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34623"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276727666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276817868Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"08704c12-507b-4ac9-86f3-7d159db139ea","remote_addr":"[fd00:1122:3344:101::b]:49698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":284,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.276837949Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"aa3fa52f-07e1-4b4a-a79a-60d978832357","remote_addr":"[fd00:1122:3344:101::c]:47890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":215,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.281407992Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f2ed1e4a-b4c3-476d-b7d7-92f5c770f99c","remote_addr":"[fd00:1122:3344:101::a]:44344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4580,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.281643357Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"567945a7-9a5f-4ada-b006-efe3cd46ded9","remote_addr":"[fd00:1122:3344:101::b]:58318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4875,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.317743422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.322731114Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.322825946Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"567a5672-659c-4ecd-bbc1-07b1cac7ebb4","remote_addr":"[fd00:1122:3344:101::c]:47890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5060,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.336048615Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"85096eb4-7419-4a89-a7c6-059975d2ab6e","remote_addr":"[fd00:1122:3344:101::c]:47890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":977,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.338867238Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42750"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.339025852Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"1c2c119e-d97a-4ac1-bcd0-c7f3d055675d","remote_addr":"[fd00:1122:3344:101::c]:42750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.383579636Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"34ec4307-9750-4e9c-afe8-cff108edd7f9","remote_addr":"[fd00:1122:3344:101::a]:34623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106993,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.383958125Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fdfa3e8a-8ef4-4e88-a565-6ba032a01bbb","remote_addr":"[fd00:1122:3344:101::a]:34623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.393204873Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50904"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.393357537Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.430877103Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.489357233Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"909540b8-4dcb-48a4-a645-582856928afa","remote_addr":"[fd00:1122:3344:101::c]:50904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96059,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:09:47.489638209Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d8ad2337-2c33-42e7-aa41-b3d54d018a0c","remote_addr":"[fd00:1122:3344:101::c]:50904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:02.579541643Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:02.579700396Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"41a79060-3a1b-4925-8488-0251cc52f1b7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40116","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:14.948744082Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"4b2edcd5-5988-49a3-b572-78800dc04f09","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7615,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.868159468Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.868273861Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ed5d7fd6-cf9c-420d-9562-cf1f1a973f81","remote_addr":"[fd00:1122:3344:101::a]:58571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.883745259Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.88376532Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ef71311e-d7d2-45d0-a647-e5907ec9c68a","remote_addr":"[fd00:1122:3344:101::a]:58571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.896695102Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58328"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.896795994Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2aa9d414-1a4f-4f17-9fe0-5977e893da21","remote_addr":"[fd00:1122:3344:101::b]:58328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.91434842Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.91437573Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"13dc40e4-4561-4f67-bbd0-e32a33a43564","remote_addr":"[fd00:1122:3344:101::b]:58328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.929813429Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c1065ca3-5c02-45d1-805c-4367455b6e63","remote_addr":"[fd00:1122:3344:101::b]:58328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":341,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.986057588Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36403"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:16.98616743Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.035542064Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101056453Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9de10361-61f4-442c-b644-f8367e779d22","remote_addr":"[fd00:1122:3344:101::b]:36403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114935,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101409981Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101536563Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101583714Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c2830f3e-ebef-4b4d-b305-b3e6570d137b","remote_addr":"[fd00:1122:3344:101::b]:36403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":253,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101613925Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61244"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101712237Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41376"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101813759Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"03769eab-3be1-4903-a98c-a7131a90196f","remote_addr":"[fd00:1122:3344:101::a]:58571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":526,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.101954393Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.102020444Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"790eb689-7a5e-4841-95bc-70f4f011e8d4","remote_addr":"[fd00:1122:3344:101::c]:41376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.102063855Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"9fdfb20b-3d3e-413e-bba5-cf78f708462b","remote_addr":"[fd00:1122:3344:101::b]:50612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":262,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.106394943Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f3d5874d-e40c-4268-828e-aa7922675d49","remote_addr":"[fd00:1122:3344:101::a]:50599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4487,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.140414161Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.180219128Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.180539126Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ccdd7f6d-556e-4f6e-843b-18eeec96d3ff","remote_addr":"[fd00:1122:3344:101::c]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.199485293Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"04c5db61-4994-4c5f-a64d-b336b9c5ab74","remote_addr":"[fd00:1122:3344:101::a]:61244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97631,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.199865302Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f1b7ce5f-3805-42ff-bce4-cd81327e7f4d","remote_addr":"[fd00:1122:3344:101::a]:61244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.222771639Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.222788839Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"599afc90-d136-406a-8d99-025eb383b056","remote_addr":"[fd00:1122:3344:101::c]:41376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.231069296Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cc939a05-0ed2-4b3c-9f8b-2dec65ea6fd6","remote_addr":"[fd00:1122:3344:101::c]:41376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.248671363Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46751"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.248762245Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.287292414Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.344361782Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"85e6c975-d616-414d-81a4-e9a74f32f478","remote_addr":"[fd00:1122:3344:101::c]:46751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95628,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:17.344599637Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"16824f56-37b9-4fa7-9853-bc2838d400d2","remote_addr":"[fd00:1122:3344:101::c]:46751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:32.612248121Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57826"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:32.612445856Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f95f40b4-6799-4015-a6eb-474c35a2e772","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57826","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:44.949659694Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"642f934c-7ab6-492d-a8f0-e0744a9b4321","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8075,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:46.965583426Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59512"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:46.965724439Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"313ae424-18b7-46d6-aafc-46355cfa483d","remote_addr":"[fd00:1122:3344:101::b]:59512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.046720937Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.046756698Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"db615f14-8147-4c41-93e9-a5f1396fd88a","remote_addr":"[fd00:1122:3344:101::b]:59512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.056729183Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a9366eb6-3705-43fb-837e-6813017fd876","remote_addr":"[fd00:1122:3344:101::b]:59512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":199,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.161129658Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47523"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.161278251Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9bdbd55b-d5c9-43b5-8d45-6715c7ac2add","remote_addr":"[fd00:1122:3344:101::a]:47523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.161898685Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51358"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.16208295Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"00d3f1b6-19ce-4139-b9b2-06f109e1ab99","remote_addr":"[fd00:1122:3344:101::a]:51358","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.170921239Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.171070362Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2c96cc4a-fa86-440e-81ce-d31ad774dd30","remote_addr":"[fd00:1122:3344:101::b]:64040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.176081505Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.176095526Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5c77e64f-e588-40af-8818-5858a8e3c461","remote_addr":"[fd00:1122:3344:101::a]:47523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.185443957Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"9f8b8fa8-fa5f-4274-800f-66df5a1308aa","remote_addr":"[fd00:1122:3344:101::a]:47523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":296,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.235170879Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60369"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.235282791Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.278132168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.338744915Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6fac9651-0bd0-43a8-8365-4243fb867122","remote_addr":"[fd00:1122:3344:101::b]:60369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103503,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.338885289Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41409"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.339119144Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.343041552Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c279ff19-b820-4fea-991e-443de2f77ad8","remote_addr":"[fd00:1122:3344:101::b]:60369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3932,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.349361475Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.34958801Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"cb5ecf21-7e13-44ec-b1d1-1a233eb80218","remote_addr":"[fd00:1122:3344:101::c]:51407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.379944795Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.398729319Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.39880029Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"662a7ed1-e623-4ae0-b5e1-ba4db16214ad","remote_addr":"[fd00:1122:3344:101::c]:51407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.407958477Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3ebf1e95-bd2e-4fb6-b826-d6ad8b68f009","remote_addr":"[fd00:1122:3344:101::c]:51407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":381,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.438749592Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.439092759Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"67bef48d-eb51-4618-9a13-5d51367dd034","remote_addr":"[fd00:1122:3344:101::c]:60086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.446170969Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a8c00615-ebe4-44a7-a2e6-da097c943f66","remote_addr":"[fd00:1122:3344:101::a]:41409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107156,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.446477996Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e78b70b5-6ae1-4d3e-9be2-c978c7147024","remote_addr":"[fd00:1122:3344:101::a]:41409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.452675876Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40158"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.452809599Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.491310628Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.550676097Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9648c25d-a9d7-492a-9214-f729080f2ce4","remote_addr":"[fd00:1122:3344:101::c]:40158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97917,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:10:47.550963294Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8a39e710-d455-4e6a-be12-b9a18183b559","remote_addr":"[fd00:1122:3344:101::c]:40158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:02.645277566Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:02.645418689Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"17b00061-5a51-4727-8aa4-a4f757de288d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36611","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:14.954504363Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"eef94e61-41c8-4975-b92d-71a074d44364","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":12912,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.809319651Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.809489895Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6eef6bb7-d539-406f-b7ca-cb2d5a7d4429","remote_addr":"[fd00:1122:3344:101::a]:38639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.834477288Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.834504949Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6f263b59-f26f-4047-a510-5500cf2024ee","remote_addr":"[fd00:1122:3344:101::a]:38639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.843317058Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c981dfb5-3da5-41f6-b0ff-5dc0b24612a5","remote_addr":"[fd00:1122:3344:101::a]:38639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.870285036Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34906"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.870362638Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d1e5b188-bb4f-419b-8d3b-3fef4c258c40","remote_addr":"[fd00:1122:3344:101::b]:34906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.93206664Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.932322996Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e4db2ecc-0e2a-4429-81e2-026e9522bb6e","remote_addr":"[fd00:1122:3344:101::a]:55927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":182,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.933006382Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.933027142Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"89f23e9c-2656-4742-9698-fa823f58e552","remote_addr":"[fd00:1122:3344:101::b]:34906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:16.945163336Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"66afb64a-89c5-414a-8e0a-ac2d03c085a6","remote_addr":"[fd00:1122:3344:101::b]:34906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":307,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.001802774Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53542"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.001935387Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.048327843Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.109408271Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"141f302d-a15d-467c-8407-9182b7e4feef","remote_addr":"[fd00:1122:3344:101::b]:53542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107540,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.109585655Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53805"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.109647897Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.109957684Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"138966d6-f520-4076-a687-d886ed0f866a","remote_addr":"[fd00:1122:3344:101::b]:53542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":289,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.109980914Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"9221474b-1211-478f-932f-3c11cd36c1d6","remote_addr":"[fd00:1122:3344:101::b]:62807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.110061086Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.130972678Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.131279395Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d7c1558f-eeae-4a18-8dc8-00d135e1758a","remote_addr":"[fd00:1122:3344:101::c]:49911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.157552328Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.179948933Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.180009614Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"38d08a20-9d57-4c28-ac4b-6f168925fb34","remote_addr":"[fd00:1122:3344:101::c]:49911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.192200879Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b188c461-9136-4906-922a-cc1310a5bba7","remote_addr":"[fd00:1122:3344:101::c]:49911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1142,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.22857484Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a7ed6ff0-1f41-4c40-b6ba-8a7365c282a8","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118647,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.228960579Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"122446ef-8a55-424a-bed3-c1ae95c4b331","remote_addr":"[fd00:1122:3344:101::a]:53805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.249184945Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48202"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.249333179Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.290287343Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.349191881Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"649f3d21-d423-430d-983c-1b3369514c93","remote_addr":"[fd00:1122:3344:101::c]:48202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99922,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.349311644Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.34955142Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2b2e48b1-7340-49f0-b27e-d006fb45dfdb","remote_addr":"[fd00:1122:3344:101::c]:48307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:17.349597761Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"376dcacf-0768-4645-93b8-2d45784ebf12","remote_addr":"[fd00:1122:3344:101::c]:48202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:32.678710798Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:32.678919703Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"66a5575e-339a-4bdb-9217-f8a72686b9b6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53301","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:44.950072469Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"5fe6c03d-6eaa-4b09-8141-040e815c0d3a","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8394,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.096591488Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.096761152Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f07a9d0e-f1ef-495a-b45d-8150eaf8aaa9","remote_addr":"[fd00:1122:3344:101::b]:57758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.116859066Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60463"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.116986919Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1998af59-3132-4241-b84c-871f6f3c671c","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.180495621Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.180530762Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2ada02b7-359e-406d-8022-bb1e0e00104f","remote_addr":"[fd00:1122:3344:101::b]:57758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.18484197Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.18486972Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6c246b21-6d13-4324-b571-8d99287e91a1","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.190526898Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"938cf0ca-fa5e-4500-add5-df183a34758b","remote_addr":"[fd00:1122:3344:101::b]:57758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.195150152Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.195323356Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3eb66d07-1c32-48ff-8d82-190f94f85c68","remote_addr":"[fd00:1122:3344:101::c]:42459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.197541846Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"598da45a-18f9-4fc8-a123-5e0b5d3ecaab","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.225998848Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38397"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.226146902Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.279078476Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.351464698Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"41c8ad12-c53b-4e59-8205-5eba02a6b4ba","remote_addr":"[fd00:1122:3344:101::a]:38397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125381,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.351727254Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53709"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.351743144Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.351804746Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a19aae06-6ee6-44bd-ae61-a629cc66a905","remote_addr":"[fd00:1122:3344:101::c]:42459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.351826766Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52709"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.35200943Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0d151a79-857b-432e-81f3-1615cd80ee3c","remote_addr":"[fd00:1122:3344:101::a]:38397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.352085832Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.358061237Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"183a12e4-87a3-46b8-bafb-37a88196d7fe","remote_addr":"[fd00:1122:3344:101::a]:53709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5955,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.362280992Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2c54f09d-faef-489b-ad26-b284b813fc3c","remote_addr":"[fd00:1122:3344:101::c]:42459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":385,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.390750184Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.39101198Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"1dbe72d0-43f1-4fd5-b78a-772f85e2ac42","remote_addr":"[fd00:1122:3344:101::b]:60040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.393900926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.426397549Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.426628414Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c3568ac0-5f42-450b-bb5e-2a238fa9375e","remote_addr":"[fd00:1122:3344:101::c]:47216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.439049184Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53096"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.458932303Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.458982284Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d6f7b135-7dde-4560-9dcf-a7c8e5e21c65","remote_addr":"[fd00:1122:3344:101::b]:52709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106945,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.463320882Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b2b9bdfd-3de4-4a5b-9f73-965bf146e9ad","remote_addr":"[fd00:1122:3344:101::b]:52709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":719,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.496748296Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.554193382Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8f3aa78e-c4d2-49f7-bc9c-b0ab24bb7cd6","remote_addr":"[fd00:1122:3344:101::c]:53096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115024,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:11:47.55455049Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9349243c-2e76-4306-a005-bf1968de57db","remote_addr":"[fd00:1122:3344:101::c]:53096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:02.711285671Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:02.711436178Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"76fe27f0-e711-4882-8345-cde6bcfb46d5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47593","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:14.951122652Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"e629e9df-e665-44d3-8e1d-ac4aac6689ac","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8690,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.841907031Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.842008374Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1debafcd-96d3-46bf-80ed-b1780b5dc6a6","remote_addr":"[fd00:1122:3344:101::a]:47768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.885765761Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.885784781Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e3a3eb1f-11ea-4a8f-a629-1f1530aa7e43","remote_addr":"[fd00:1122:3344:101::a]:47768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.895179213Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"bb1feed7-9301-4c47-b5a5-c82ab102ecb4","remote_addr":"[fd00:1122:3344:101::a]:47768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.930267305Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.930372807Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"10686b0e-531e-4606-a597-4db927143e01","remote_addr":"[fd00:1122:3344:101::b]:42090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.950149544Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43400"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:16.950280717Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.004369317Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081271962Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b971616c-f822-4829-bb6e-bd113d07a5f8","remote_addr":"[fd00:1122:3344:101::a]:43400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131035,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081432876Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33151"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081496947Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43378"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081536468Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60105"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081829215Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081872176Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5d79d3f9-5a45-4532-8555-0f00d4c453ef","remote_addr":"[fd00:1122:3344:101::b]:42090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":289,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081932917Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"109abd0f-ad7e-4407-8767-5f0df7ad8e34","remote_addr":"[fd00:1122:3344:101::b]:60105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":306,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.081955178Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1e96d346-003b-4bae-9816-4c6d4ad513a2","remote_addr":"[fd00:1122:3344:101::a]:43400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":378,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.082014569Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.082095341Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0214526a-8a28-4d82-b92c-9b030fb682e7","remote_addr":"[fd00:1122:3344:101::a]:33151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":347,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.093542729Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.093715923Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"752c5c99-6954-458e-be60-a24c27f086ab","remote_addr":"[fd00:1122:3344:101::c]:43802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.096838163Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"880b5f9d-bc6b-48c5-a138-09d1bb9ec182","remote_addr":"[fd00:1122:3344:101::b]:42090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.129747886Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.139163848Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.139211379Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d208631d-7f94-4f4c-88cf-5f82571ab360","remote_addr":"[fd00:1122:3344:101::c]:43802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.149296047Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e9599602-17bf-4f79-a8c4-306086910b88","remote_addr":"[fd00:1122:3344:101::c]:43802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":360,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.191257674Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.191498569Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5795d1dc-a767-475e-b666-15ae6a74c432","remote_addr":"[fd00:1122:3344:101::c]:35117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.193124596Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0cc9bcba-fe8a-4874-bb91-d73ab0cda340","remote_addr":"[fd00:1122:3344:101::b]:43378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111297,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.193423873Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8aaae324-5020-4e0f-b95b-ee82b24eb098","remote_addr":"[fd00:1122:3344:101::b]:43378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.220435682Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56678"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.220556005Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.258515721Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.319147629Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"36b36bc6-82ec-4e88-89eb-6324c194ef44","remote_addr":"[fd00:1122:3344:101::c]:56678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98635,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:17.319430325Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"549ce7ae-4c7f-4ee4-8d7d-49dc0b62a2b6","remote_addr":"[fd00:1122:3344:101::c]:56678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:32.743586389Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:32.743723362Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"75709318-27eb-4c1d-8c1b-20fbae33955d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39744","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:44.950770524Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"227a8e37-0619-4e9f-96e0-753fcf95f14c","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8568,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:46.952632919Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:46.952867795Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6a753583-24d0-42f4-a8bb-17e501f6c653","remote_addr":"[fd00:1122:3344:101::a]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.016791507Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51509"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.020557782Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"28e9e102-cd18-4003-ab3e-fd1daaa60df4","remote_addr":"[fd00:1122:3344:101::b]:51509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.06836051Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.068400001Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"81ea0824-e66a-4456-bd5a-c732989ab473","remote_addr":"[fd00:1122:3344:101::a]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.084627507Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cf4bbd5c-40c4-45b4-99a4-093649510d93","remote_addr":"[fd00:1122:3344:101::a]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":374,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.089670491Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.089701832Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fffecbd0-67c5-4490-b438-c5b4857cd8ee","remote_addr":"[fd00:1122:3344:101::b]:51509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.102118942Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cb586673-1f37-49d9-ab2a-e3686f72593e","remote_addr":"[fd00:1122:3344:101::b]:51509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":334,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.209923583Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58197"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.210086036Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"cb642a7b-25fc-4420-a428-28552196f878","remote_addr":"[fd00:1122:3344:101::a]:58197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.249278512Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35419"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.249359514Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"703b66bb-2b61-42ca-a5d5-19cd9d9f8605","remote_addr":"[fd00:1122:3344:101::c]:35419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.261230292Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.261392625Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"6422cd4a-b58b-49fe-8679-0ba729edf59b","remote_addr":"[fd00:1122:3344:101::b]:47200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.264514476Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.264531516Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a10fc6c2-eac5-49ea-97fb-4d6d91b862d2","remote_addr":"[fd00:1122:3344:101::c]:35419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.291136796Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.29127694Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8a2bd2e4-9b14-4715-8b2c-e6240fe44885","remote_addr":"[fd00:1122:3344:101::c]:38758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.294681067Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e7f34e3c-8166-4bee-baad-0d983f2defb8","remote_addr":"[fd00:1122:3344:101::c]:35419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.294709747Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48714"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.294786879Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.333572294Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.391425899Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"721fa2c6-387a-47c6-8122-ccae536096a3","remote_addr":"[fd00:1122:3344:101::a]:48714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96663,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.391696585Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42595"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.391742146Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61503"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.391980422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.392014282Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a12e1d69-70f5-4b14-8832-e1ad685e92c6","remote_addr":"[fd00:1122:3344:101::a]:48714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":256,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.431093914Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.487171529Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.487240131Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"90ea162f-b498-4c6a-b3be-887af3867b5d","remote_addr":"[fd00:1122:3344:101::b]:42595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95334,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.491376104Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fc3acc26-d351-47d5-846c-1b318fc8972e","remote_addr":"[fd00:1122:3344:101::b]:42595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":738,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.521071394Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.581211911Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1f64b0ad-bb38-4b2d-8a98-9da91e9937b3","remote_addr":"[fd00:1122:3344:101::c]:61503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189240,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:12:47.581654041Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"929f7d03-8dde-4009-8ddd-4c4973ef6b76","remote_addr":"[fd00:1122:3344:101::c]:61503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:02.775626542Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40430"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:02.775762055Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1cae164c-3ae7-4b39-80dd-6b104ba52ded","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40430","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:14.949898623Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"8549c095-0389-44b3-93b9-8b5a5fb93f25","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7800,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.772486823Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.772630276Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5b5137f2-8a7e-4086-8c55-a8b4524a81a1","remote_addr":"[fd00:1122:3344:101::b]:56439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.8348368Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.83486402Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0465739b-2814-43ca-9b24-d2534c9a6d67","remote_addr":"[fd00:1122:3344:101::b]:56439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.844445137Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f225b658-d4c8-4eda-b4e4-aa442ea99179","remote_addr":"[fd00:1122:3344:101::b]:56439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":226,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.878103056Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.878192508Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"57de6fce-dbfb-45a9-9620-d1a774e4de1a","remote_addr":"[fd00:1122:3344:101::a]:55194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.916254326Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.916456491Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ab204cc4-fe18-4c97-aeb5-ab972e859728","remote_addr":"[fd00:1122:3344:101::a]:57907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.933286751Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.933422474Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7b64ab6b-bd25-4268-af00-2636b5cdde2e","remote_addr":"[fd00:1122:3344:101::b]:49071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.938858116Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43268"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.938934018Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:16.987213607Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.06708607Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7640a2fd-8f75-4ea4-95e0-a3b595ef9e56","remote_addr":"[fd00:1122:3344:101::b]:43268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067228453Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61650"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067326265Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41686"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067636262Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067659982Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"115b3841-1766-4f20-8ec0-62baba45dcc2","remote_addr":"[fd00:1122:3344:101::c]:41686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067693713Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c836bc17-0e8f-4e6c-b23a-bc683edde085","remote_addr":"[fd00:1122:3344:101::a]:55194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":277,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067817316Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ca490e77-d3ff-4c2b-b0d9-5b8f611431b8","remote_addr":"[fd00:1122:3344:101::b]:43268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":372,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.067835266Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.081491045Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"33bde1d6-ebd7-4569-aab5-90b839166873","remote_addr":"[fd00:1122:3344:101::a]:55194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":534,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.11319542Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.119098203Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.119149514Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"bd57f15c-5eb5-42b6-9dc3-fa1da1575bda","remote_addr":"[fd00:1122:3344:101::c]:41686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.129501308Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"017acc11-827d-4e4a-a86d-246f1e9ed623","remote_addr":"[fd00:1122:3344:101::c]:41686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":698,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.171054275Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42591"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.172551429Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.172568349Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"80c57f8f-d866-4a94-adb8-36686d24487f","remote_addr":"[fd00:1122:3344:101::a]:61650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104903,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.177511421Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9021c79c-4406-4855-8307-c37a0f111b48","remote_addr":"[fd00:1122:3344:101::a]:61650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1007,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.21559426Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.231164222Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.231586061Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"295827d2-4d47-419a-9141-c2a8c1c5a1e5","remote_addr":"[fd00:1122:3344:101::c]:56367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.274423087Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d32e0502-b6b4-4781-85d8-37e1edc9c4ca","remote_addr":"[fd00:1122:3344:101::c]:42591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103234,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:17.274749085Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"518fce47-0600-44cc-aac8-dac92a677398","remote_addr":"[fd00:1122:3344:101::c]:42591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:32.808310917Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:32.808490371Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"91064276-afbe-4a48-a3ce-f43c921d5a26","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35582","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:44.952208094Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"997464a0-a54a-4e20-a758-27504cc52375","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9543,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.115553583Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.115662375Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"fd297be6-d94b-4341-a5b2-3f3249e61cdb","remote_addr":"[fd00:1122:3344:101::a]:54537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.145753514Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.145858697Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d1695704-5194-46e5-8d4c-1fa124a2c7fc","remote_addr":"[fd00:1122:3344:101::b]:33767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.170536584Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.170565874Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f37552ec-858c-4da5-9f69-5b909f401c4e","remote_addr":"[fd00:1122:3344:101::a]:54537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.179744561Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"02113678-50a0-4c2a-85ba-c65463be9622","remote_addr":"[fd00:1122:3344:101::a]:54537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.201577844Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.201600174Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"095a0eaa-88c3-4ef5-a34a-cd45db32cb6d","remote_addr":"[fd00:1122:3344:101::b]:33767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.283110553Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.283283248Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ea92d14a-3915-456d-8750-bc9b59512404","remote_addr":"[fd00:1122:3344:101::b]:36912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.29227439Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6b6a77ba-72a6-4298-bbc4-6651ae13e3bc","remote_addr":"[fd00:1122:3344:101::b]:33767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.320330163Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46659"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.320429835Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.320456576Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.320466806Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.320928797Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.331394763Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36186"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.331675099Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.336928098Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.337067121Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.368788737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.369130004Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.380351927Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.390871885Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34745"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.396239196Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49591"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.401257036Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.401360322Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.406931367Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"839594ed-c238-4b94-901b-7c3977215223","remote_addr":"[fd00:1122:3344:101::c]:49591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":542,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.40704718Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0419581f-b654-4f04-a725-9753cce099eb","remote_addr":"[fd00:1122:3344:101::a]:34745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":678,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.431213245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.431576093Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.448654139Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51690"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.458220734Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.458278036Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0338b90e-9ec9-4e98-ac2d-44cbbb13d0fd","remote_addr":"[fd00:1122:3344:101::c]:49591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.460326052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.460381553Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3651eecb-e5fb-45ff-ab50-b6c080ba85fb","remote_addr":"[fd00:1122:3344:101::b]:36186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128818,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.464418674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.464559807Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.46954503Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b84a98dd-d473-4072-ba2e-1680ff91231d","remote_addr":"[fd00:1122:3344:101::b]:36186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4780,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.470534832Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"06960b3d-99dd-463c-9fa9-677503df6f7b","remote_addr":"[fd00:1122:3344:101::c]:49591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":894,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.493635953Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.493705035Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.500224152Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.504163991Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63361"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.504803805Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.504885607Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.504950509Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.50500819Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.505057821Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.505090432Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.505180984Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.505226785Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b9f87dcb-f07b-493a-849f-fc1a8137d133","remote_addr":"[fd00:1122:3344:101::b]:63361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":831,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.510633797Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51535"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.510929264Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.510963024Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.511075117Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.511586378Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.521184265Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.521724577Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.540954521Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.541052803Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.545184066Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.545626856Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4a2f3de7-5008-4644-b251-6611514b0b89","remote_addr":"[fd00:1122:3344:101::c]:39289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.557434563Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39768"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.573576417Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.573648868Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.580976414Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.581022815Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"286257fc-4d29-455f-866e-b0b3068d73ea","remote_addr":"[fd00:1122:3344:101::c]:51690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132128,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.588543875Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"83e79c44-6a38-44df-b715-0690a4a32fd3","remote_addr":"[fd00:1122:3344:101::c]:51690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":853,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.588669978Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.588686828Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.609056747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.609255162Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.620970596Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65117"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621548919Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621622621Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621675702Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621712093Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621734773Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621759194Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621777514Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.621819885Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"53724ecc-828d-4f45-9959-2040d28d543a","remote_addr":"[fd00:1122:3344:101::c]:65117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":652,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.626097302Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.626114612Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.630665185Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.649602712Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"cd2a33cc-fbe3-4f4e-9d5f-404a5f4ba54b","remote_addr":"[fd00:1122:3344:101::b]:46659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":329151,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.652275373Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.652302513Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.681640925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.681704106Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.702129697Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0c0345d4-3522-432d-b5ab-7adfc812380d","remote_addr":"[fd00:1122:3344:101::a]:39768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144516,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.70269209Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2cf39189-df7b-4cf4-8967-d17425479557","remote_addr":"[fd00:1122:3344:101::a]:39768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":287,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.709981024Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.710001165Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.733418473Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.733486395Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.759033251Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.759085342Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.782096331Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.782145423Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.794927111Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39024"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795249608Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5082dfc8-feb0-4452-a8c4-3c0037101230, kind: Service { id: 71878588-3bc5-4be9-bddb-893f09285184 }, name: Name(\"external-dns-71878588-3bc5-4be9-bddb-893f09285184\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 84c8b3b4-9160-4f8a-9f93-e97bdb5fa721, kind: Service { id: 870344ce-2728-444f-8d5a-e3771e15477e }, name: Name(\"external-dns-870344ce-2728-444f-8d5a-e3771e15477e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 234])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 32dc4ecd-b53e-48ec-96da-e27f9d50a7c1, kind: Service { id: cffb0aad-1932-4a44-891c-2739991d1934 }, name: Name(\"nexus-cffb0aad-1932-4a44-891c-2739991d1934\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 193])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5bb6cca4-8c89-42c7-8517-bbec55a75150, kind: Service { id: 5236fac0-d91c-403e-8658-32167d628d4d }, name: Name(\"nexus-5236fac0-d91c-403e-8658-32167d628d4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 106])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7464f2e2-2881-4143-9c24-14f2de0a4d6a, kind: Service { id: bc5aeb1b-883b-4e3c-84e4-bb4b98458327 }, name: Name(\"nexus-bc5aeb1b-883b-4e3c-84e4-bb4b98458327\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 156])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795298839Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.79533222Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795359791Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795375131Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795393371Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795413232Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.795441923Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b636a91a-1186-4693-ad23-1ec6c1b45dad","remote_addr":"[fd00:1122:3344:101::a]:39024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":400,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:47.79752928Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"ad7ce938-b192-4aab-9687-03eddbc9a99b","remote_addr":"[fd00:1122:3344:101::a]:51535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286755,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.584766841Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65092"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.584905084Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.584911955Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.584954936Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.585207831Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 4eca6aeb-4f07-4f83-9a97-c0923ef06389 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.598303007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.598344578Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_4eca6aeb-4f07-4f83-9a97-c0923ef06389"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 8ae4f12d-2c4d-4d52-9800-1f43946d3827 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.625728245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.625770906Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_8ae4f12d-2c4d-4d52-9800-1f43946d3827"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: eb3af0a6-1a5d-4c13-b73f-b02889253b79 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.649791319Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.649826757Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_eb3af0a6-1a5d-4c13-b73f-b02889253b79"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: c9b2f2eb-a892-4cf7-8906-3469787aaff7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.672142063Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.672189054Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_c9b2f2eb-a892-4cf7-8906-3469787aaff7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.695347546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.695383237Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: a132b3b6-342d-41c2-a6be-600f78f98f46 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.717446745Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.717489386Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_a132b3b6-342d-41c2-a6be-600f78f98f46"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 21d10cf0-acfa-456c-8810-1f3a2325e0aa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.738114731Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.738151012Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_21d10cf0-acfa-456c-8810-1f3a2325e0aa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e7d81b5e-d5f4-4518-a5fd-45186798cf74 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.759113655Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.759155506Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e7d81b5e-d5f4-4518-a5fd-45186798cf74"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7cdbd7a4-396c-471d-9a8a-b2f8c166fe95 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.780115319Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.78015999Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7cdbd7a4-396c-471d-9a8a-b2f8c166fe95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:13:48.792063238Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"efe792a5-051a-440b-b349-1f1f54543b3a","remote_addr":"[fd00:1122:3344:101::c]:65092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:02.844570696Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:02.84474295Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"92fa5aa5-4a96-4aeb-b658-d928f5a83d75","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48433","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:14.951972927Z","hostname":"buskin","pid":1566,"uri":"/66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","method":"GET","req_id":"42de204d-e588-4e2f-aa3f-4620ce122638","remote_addr":"[fd00:1122:3344:101::d]:63574","local_addr":"[fd00:1122:3344:101::1]:40239","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9473,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.786904917Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.787014849Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a0e9cde7-6b04-44d6-8c73-fcfb276bd067","remote_addr":"[fd00:1122:3344:101::b]:53547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.825988018Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.826175032Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"fa589104-6d03-4475-a53b-0955af262fa9","remote_addr":"[fd00:1122:3344:101::a]:41846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.829369555Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:874","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(4929565) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.830563351Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"PUT","req_id":"bc2941fe-544e-45c2-bc7a-acd055879747","remote_addr":"[fd00:1122:3344:101::a]:41846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4148,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.834244385Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {3f150d20-bde0-4a89-bf9c-3cc8293a508d: {10.151.1.175: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}, 10.151.1.176: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}}} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.836505725Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.836515776Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fbe43c24-5440-45d2-bdde-63d2f69a240d","remote_addr":"[fd00:1122:3344:101::b]:53547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2480,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.845451027Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e8490a27-5b69-4629-a4df-ba5225de62c2","remote_addr":"[fd00:1122:3344:101::b]:53547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.887513406Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.887621269Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c00a2f00-0bd4-4b48-b007-6654eb720d95","remote_addr":"[fd00:1122:3344:101::a]:37271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.898702449Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.898873253Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"af2473cc-c8b8-4c7f-9d7f-fb4075f5da23","remote_addr":"[fd00:1122:3344:101::b]:55493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.941999096Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {3f150d20-bde0-4a89-bf9c-3cc8293a508d: {10.151.1.175: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}, 10.151.1.176: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.942770213Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"20224928-ca8a-4dc3-97c8-5840f5eeecf0","remote_addr":"[fd00:1122:3344:101::a]:37271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":855,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.957858134Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62392"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:16.957973476Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.006538112Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.079841686Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cc6b4dfb-cf5b-43ba-9368-8f8caded9051","remote_addr":"[fd00:1122:3344:101::b]:62392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121913,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.080153573Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.080542362Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f01fc874-265b-4db2-ab19-fc64d721408f","remote_addr":"[fd00:1122:3344:101::b]:62392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":314,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.080557072Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.085926693Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3b33dbdd-4ada-414d-aeb9-31ff3a1d5ad9","remote_addr":"[fd00:1122:3344:101::a]:37271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5654,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.124138405Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.13674771Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33706"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.137084877Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d16a2776-5aec-4b2e-9319-e759c6982674","remote_addr":"[fd00:1122:3344:101::c]:33706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.183487034Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.183795971Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e1f29afb-1f53-4d6b-8ecf-0df20e4b04a7","remote_addr":"[fd00:1122:3344:101::c]:47694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.207034495Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5f3c8454-4407-4b72-9da7-03676f9d98ca","remote_addr":"[fd00:1122:3344:101::a]:53216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126635,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.207423924Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"adc79eec-b7e3-474f-9a35-527d0018214a","remote_addr":"[fd00:1122:3344:101::a]:53216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.214152396Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.214370461Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e520a943-3aae-422d-836e-19818d030c8e","remote_addr":"[fd00:1122:3344:101::a]:48907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.230384472Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {3f150d20-bde0-4a89-bf9c-3cc8293a508d: {10.151.1.175: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}, 10.151.1.176: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.230413853Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"57c7ded4-7225-47bf-8a78-3290ee99003d","remote_addr":"[fd00:1122:3344:101::c]:47694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.240230324Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"564e3b16-8590-47e9-bdf9-f001db68a96a","remote_addr":"[fd00:1122:3344:101::c]:47694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.24933492Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40712"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.249429752Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.290959239Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.351885403Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3c52849c-4fd3-46a3-99f3-cc23c03aeb97","remote_addr":"[fd00:1122:3344:101::c]:40712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102488,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.352010786Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.352290873Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"bb61d942-44a9-481a-baf6-51cd9d06c6a1","remote_addr":"[fd00:1122:3344:101::c]:40712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.370821281Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 795528e9-4767-49c0-875c-03687d636f7a, project_id: 0918b7c8-92fc-42db-bc18-6b29244686c9 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-09T01:14:17.253394501Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(4929565))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: de64f350-6916-4aa4-a3c1-c29e05f64a1e, slot: 0, read_only: false, vcr_json: .., name: \"disk-f0ac5086f4f\" }], boot_settings: Some(InstanceBootSettings { order: [de64f350-6916-4aa4-a3c1-c29e05f64a1e] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.370859541Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.370875402Z","hostname":"buskin","pid":1566,"propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:1562","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(4929565))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3f150d20-bde0-4a89-bf9c-3cc8293a508d, kind: Instance { id: d2d2bb3c-2257-4f1d-9c60-defe7d956daa }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(4929565), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: de64f350-6916-4aa4-a3c1-c29e05f64a1e, slot: 0, read_only: false, vcr_json: .., name: \"disk-f0ac5086f4f\" }], boot_settings: Some(InstanceBootSettings { order: [de64f350-6916-4aa4-a3c1-c29e05f64a1e] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-09T01:14:17.253394501Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.371547887Z","hostname":"buskin","pid":1566,"uri":"/vmms/cc75425d-2d9d-4046-87bc-6a23a79c7a17","method":"PUT","req_id":"6baa48a6-61b5-4eec-b8b0-cf123c889e86","remote_addr":"[fd00:1122:3344:101::a]:43300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":19427,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.485778154Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42760"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.490201504Z","hostname":"buskin","pid":1566,"uri":"/vmms/cc75425d-2d9d-4046-87bc-6a23a79c7a17/state","method":"PUT","req_id":"1847bf43-cee4-4a7c-ae0b-adebe0d23e2d","remote_addr":"[fd00:1122:3344:101::a]:42760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4294,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.492807073Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 4929565 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.493962349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.54056521Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.565336189Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.595869048Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.596050272Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"dc5874aa-4f71-4b6c-8a58-f9daa4f1f3eb","remote_addr":"[fd00:1122:3344:101::a]:34881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.638351367Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {3f150d20-bde0-4a89-bf9c-3cc8293a508d: {10.151.1.175: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}, 10.151.1.176: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.638402738Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"427eb0fc-d12d-4adc-bb89-caf8aa210eda","remote_addr":"[fd00:1122:3344:101::a]:34881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:17.722997026Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"99b751a6-487d-4b01-acd7-751ee9e0042e","remote_addr":"[fd00:1122:3344:101::a]:34881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":684,"response_code":"204"} {"msg":"Profile for oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:19.139950215Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:19.140420465Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 41.727422ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:19.345209886Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 73.804062ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:19.398972009Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 275.422311ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:19.488405016Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 532.318811ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:19.784511847Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 465.757801ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:20.334158038Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 1.118475934s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:20.81336758Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 1.020151118s","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:21.945313939Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 924.683592ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:22.977823744Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\") failed: Property not found. retry in 912.530406ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:23.914099178Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:24.856806597Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:2067"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:24.865977924Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:2076"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:24.904207796Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:2100"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.055398717Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:1213"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2025-01-09T01:14:25.056579234Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.056759288Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.056773518Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-09T01:14:25.056579234Z }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.056780519Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-09T01:14:25.056579234Z }, migration_in: None, migration_out: None }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2025-01-09T01:14:25.117275143Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.117291784Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.117311494Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-09T01:14:25.117275143Z }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:25.117331785Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-09T01:14:25.117275143Z }, migration_in: None, migration_out: None }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:32.879109612Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64613"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:32.879310187Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"cccd5814-d9a8-44d7-b40b-c5b8c370a6bf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64613","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.045378587Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:32768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.046269347Z","hostname":"buskin","pid":1566,"uri":"/vmms/cc75425d-2d9d-4046-87bc-6a23a79c7a17/state","method":"PUT","req_id":"b19e947c-c51f-451f-bcf2-3421f874b638","remote_addr":"[fd00:1122:3344:101::a]:32768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":709,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2025-01-09T01:14:35.046388069Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.04641051Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.046426421Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-09T01:14:35.046388069Z }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.046439691Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-09T01:14:35.046388069Z }, migration_in: None, migration_out: None }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2025-01-09T01:14:35.091511078Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.091530118Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.091549348Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2025-01-09T01:14:35.091511078Z }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.091558929Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.09160055Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:35.091680862Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"13159395221909322921"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:35.091726503Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"9612014113215945717"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.091892146Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.092566151Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/3d778ce3-4cf1-4708-abb3-05822320be17/debug/bundle/zone\", \"/pool/int/bf8eca8a-81d6-48ff-9487-7cf623d74763/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/4eca6aeb-4f07-4f83-9a97-c0923ef06389/crypt/debug\", \"/pool/ext/8ae4f12d-2c4d-4d52-9800-1f43946d3827/crypt/debug\", \"/pool/ext/eb3af0a6-1a5d-4c13-b73f-b02889253b79/crypt/debug\", \"/pool/ext/c9b2f2eb-a892-4cf7-8906-3469787aaff7/crypt/debug\", \"/pool/ext/1d6f1a4e-aae6-43a7-bf8c-e8433f676c4f/crypt/debug\", \"/pool/ext/a132b3b6-342d-41c2-a6be-600f78f98f46/crypt/debug\", \"/pool/ext/21d10cf0-acfa-456c-8810-1f3a2325e0aa/crypt/debug\", \"/pool/ext/e7d81b5e-d5f4-4518-a5fd-45186798cf74/crypt/debug\", \"/pool/ext/7cdbd7a4-396c-471d-9a8a-b2f8c166fe95/crypt/debug\"] }","zone_name":"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:35.740776686Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17\", bundle_id: 3fa96058-45c6-4566-8f8f-86711ee469cd }, time_created: 2025-01-09T01:14:35.093430811Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_cc75425d-2d9d-4046-87bc-6a23a79c7a17","v":0,"name":"SledAgent","level":40,"time":"2025-01-09T01:14:35.740854467Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:1313"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.514151886Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.531796354Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(4929565), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.531826425Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(4929565), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.53381262Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-09T01:14:37.533784049Z }, migration_in: None, migration_out: None }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.621841716Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-09T01:14:37.533784049Z }, migration_in: None, migration_out: None }","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:37.627626896Z","hostname":"buskin","pid":1566,"instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","propolis_id":"cc75425d-2d9d-4046-87bc-6a23a79c7a17","instance_id":"d2d2bb3c-2257-4f1d-9c60-defe7d956daa","component":"InstanceManager","file":"sled-agent/src/instance.rs:1250"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.098991001Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.099130634Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"422dc3a5-b972-4862-8305-aca9b29fd721","remote_addr":"[fd00:1122:3344:101::c]:45858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.119366951Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {3f150d20-bde0-4a89-bf9c-3cc8293a508d: {10.151.1.175: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}, 10.151.1.176: {205cb22a-cb2b-4dd5-b8ed-92ee79f9f372}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.119393992Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fbf7c631-7298-4db5-8045-900302a89daf","remote_addr":"[fd00:1122:3344:101::c]:45858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.128488377Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"710aa22a-a41c-4d50-a54f-a615fee79655","remote_addr":"[fd00:1122:3344:101::c]:45858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":296,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.306139945Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:38.306346899Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ada4060a-65e8-463c-a1f3-f0bf9b4338c4","remote_addr":"[fd00:1122:3344:101::c]:65396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.549425785Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.549674721Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"a0757a7d-a0c3-4bb3-b1bc-1a3aaffcc244","remote_addr":"[fd00:1122:3344:101::a]:57557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.718375297Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.718549651Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"73e6c449-2d1f-4d82-8332-c2f73c4fc84a","remote_addr":"[fd00:1122:3344:101::a]:34783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.738747507Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.738832559Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.738839209Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7846b36c-d685-411c-95f1-eb5ceb74be33","remote_addr":"[fd00:1122:3344:101::a]:34783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:39.749833317Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a7580b3b-a524-4586-8620-a15af07c9320","remote_addr":"[fd00:1122:3344:101::a]:34783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":382,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:40.321622618Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:40.321752331Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d7d6061d-1758-4bef-8a81-c4743f8d0675","remote_addr":"[fd00:1122:3344:101::a]:48443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:40.36388334Z","hostname":"buskin","pid":1566,"sled_id":"66e03bc8-0dd2-4f2a-93c0-ce1c4f2f1e4a","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:40.363908531Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"14e327c4-82af-4dcb-b10c-d5a2f5971525","remote_addr":"[fd00:1122:3344:101::a]:48443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:40.37584754Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fd3a135e-953c-4865-9a18-b6216f7f4d26","remote_addr":"[fd00:1122:3344:101::a]:48443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":255,"response_code":"204"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-09T01:14:41.056428736Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}