[ Oct 11 15:59:21 Enabled. ] [ Oct 11 15:59:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 11 15:59:21 Method "start" exited with status 0. ] [ Oct 11 15:59:21 Rereading configuration. ] [ Oct 11 15:59:21 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.117993042Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.118145624Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.169381805Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.169429856Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.283690903Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.292691221Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.300765713Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:21.901240991Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038580504Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038669315Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038735587Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038827108Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038842778Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.038878179Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.041919112Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:22.041925343Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.041938643Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.041950923Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.041965763Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.042006144Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.042079275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.042148436Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.042164087Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.053092689Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.053121399Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"c8821193-671b-45a0-9a99-7032783dba68 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.151204152Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.556313189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.55634588Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.55635459Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.55637185Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.55637866Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.556400561Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.55640615Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.56488591Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.56490949Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:22.654913301Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052797691Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052834381Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052872042Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052880182Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052894052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052908372Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052947253Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052954773Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052975054Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052981414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.052987034Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053004254Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053009644Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053014654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053019965Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053039765Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053046465Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053051395Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053055495Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053059615Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053077395Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053092016Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053097696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053101986Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053107666Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053112796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053119076Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053123866Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053144997Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053151267Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053155237Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053159617Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053190928Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053196258Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053204498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053208908Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053213808Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053219358Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053250929Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053258519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053265109Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053269529Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053274259Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053279899Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053284889Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053293139Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.05332909Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.05333474Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.05333899Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.05334421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.0533485Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.05335351Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053388421Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053395951Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053400311Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053411841Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053416332Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053448112Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053453472Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053457672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053461952Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053466412Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053485183Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053493763Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053505783Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053513043Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053517923Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053547694Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.053557694Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.063923256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.063968767Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999925518s","next_cleanup":"Instant { tv_sec: 2277, tv_nsec: 60818137 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.064017828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.064028058Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3944"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.064067248Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.064076059Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.066950969Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.068954774Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069159538Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069169058Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069211139Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.06924919Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.06927377Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.06927839Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.06930106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069310181Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069317721Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069322101Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069325641Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069331881Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069353612Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069380872Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069649907Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: d30c9cf6-9f94-4f76-affa-4751a592517c,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: d0c143e7-a173-42d2-843a-c5f94a0fd260 (sled),\n rack_id: d30c9cf6-9f94-4f76-affa-4751a592517c,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069903481Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069907581Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069960172Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.069972642Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.070131585Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.070243397Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.070289638Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d0c143e7-a173-42d2-843a-c5f94a0fd260 (sled), rack_id: d30c9cf6-9f94-4f76-affa-4751a592517c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.070717125Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4241"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.070784026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4623"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.075423608Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.081255281Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.081422244Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085509655Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085578246Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085701469Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:42358","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d0c143e7-a173-42d2-843a-c5f94a0fd260 (sled), rack_id: d30c9cf6-9f94-4f76-affa-4751a592517c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.08580078Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085818141Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085854551Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085868682Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085875232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085880282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085927593Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085945433Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085979784Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085991624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.085998854Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.086005814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.086018614Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.086027004Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.101786881Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.122023657Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.140034383Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.17457161Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:391"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.174616561Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.174622011Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:403"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.174626421Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 67.240617ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.177558622Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"581f888c-742b-47b1-b046-8b2cf2f2a2e0\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"581f888c-742b-47b1-b046-8b2cf2f2a2e0\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_c8821193-671b-45a0-9a99-7032783dba68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.188314451Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_c8821193-671b-45a0-9a99-7032783dba68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.212372484Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_c8821193-671b-45a0-9a99-7032783dba68/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.212417205Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.2178196Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:418"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.2178369Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_c8821193-671b-45a0-9a99-7032783dba68/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.21784308Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.245783181Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 62.987455ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.24632228Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b6850f42-38bf-44e4-8573-b03208406e2e\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b6850f42-38bf-44e4-8573-b03208406e2e\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.305157554Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.310257584Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 214.324673ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.310832294Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15b64186-ddc7-492f-ae45-51b05cc5ab93\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"15b64186-ddc7-492f-ae45-51b05cc5ab93\" }"} {"msg":"Mounting oxi_c8821193-671b-45a0-9a99-7032783dba68/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.524343904Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.526584644Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 110.547544ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.52753192Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"326d8f15-d2f9-4d43-b40b-8b7ca9c58a3b\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"326d8f15-d2f9-4d43-b40b-8b7ca9c58a3b\" }"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.536998076Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.537070788Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.537087288Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.537122039Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.638641132Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 191.540615ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.639327244Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92c7077b-90da-4bcc-8b93-7a44eb74fc17\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92c7077b-90da-4bcc-8b93-7a44eb74fc17\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.832385786Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 190.266967ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:23.833015327Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b65b0faf-9f1a-411b-acc6-042024816dfd\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b65b0faf-9f1a-411b-acc6-042024816dfd\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.023590514Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 175.664339ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.024112003Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea01baf4-5c00-4e4d-abdc-7417ac2f938f\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea01baf4-5c00-4e4d-abdc-7417ac2f938f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.201184554Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 443.847867ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.201629511Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5aa73caf-34cb-4206-b8f2-b1c8cb15fb6d\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 15:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5aa73caf-34cb-4206-b8f2-b1c8cb15fb6d\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.275412318Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.280517057Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.280557618Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.280620319Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:47594","producer_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:24.281704868Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"261.188504ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.282549083Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.282555483Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.282615194Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.282620884Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:24.544283971Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"658.642114ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:24.64727023Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:25.204771633Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.192658372s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:26.398465463Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.742575352s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659309045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2699"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659325816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2917"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659341706Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2569"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659354906Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2616"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659379397Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2998"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.659397367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3040"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.661242409Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2842"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.66243107Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2610"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.662542162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:26.662675135Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 27.522881ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:26.877256264Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 141.003702ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:27.063294352Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 268.153719ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:27.244882802Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 475.165116ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:27.616001202Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.199065587s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:28.243194249Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:28.244101165Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.954968171s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 719.867054ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:29.488985354Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.388217194s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:30.370328996Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.436261039s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:31.941452736Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:33.201235156Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.982975273s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668288821Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668320682Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668346252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668360152Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668629367Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.66882307Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668890421Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668923642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3944"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.668928352Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:42358","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.896901307Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.896983248Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.89710133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.897173842Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.897200322Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.897241153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.897292344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.897323074Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.902271621Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4265","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.902283972Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4281"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:33.902293302Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.109691814Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.137518263Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60245"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.137835079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.137979411Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"eb7d9bc6-f2f0-478d-a276-3e6860e7bb9f","remote_addr":"[fd00:1122:3344:101::1]:60245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":273,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.167432359Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34583"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.167778805Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"6b79ca64-5f21-4f29-82ff-0572a928d7e3","remote_addr":"[fd00:1122:3344:101::1]:34583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.16979502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.169827021Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.170225708Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.170611234Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:884"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: b9e10068-47f0-454a-a7ca-7f0be2a2cc2b, pool_id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 207ec097-7a59-4b17-bdb0-fc1e09b7d155, pool_id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1560c5d9-be44-4c1a-a253-e9c088fb614e, pool_id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: dab496d0-1cbb-402c-bbbe-e2ba6f1827c1, pool_id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 01f83159-6b1d-4c60-a62f-7506f371d2c5, pool_id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: d37cfd42-08c5-4d18-8e74-11ce5ce654b0, pool_id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: d7095ee2-a8ea-4e52-a9e7-10195cfeb63c, pool_id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3baaebf4-e07b-46c6-a2af-5bd598770095, pool_id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: d4092a69-3126-47bf-99a5-32b743b7cd67, pool_id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.20278887Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.202903772Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:63717"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203202367Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: b9e10068-47f0-454a-a7ca-7f0be2a2cc2b, pool_id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 207ec097-7a59-4b17-bdb0-fc1e09b7d155, pool_id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 1560c5d9-be44-4c1a-a253-e9c088fb614e, pool_id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: dab496d0-1cbb-402c-bbbe-e2ba6f1827c1, pool_id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 01f83159-6b1d-4c60-a62f-7506f371d2c5, pool_id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: d37cfd42-08c5-4d18-8e74-11ce5ce654b0, pool_id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: d7095ee2-a8ea-4e52-a9e7-10195cfeb63c, pool_id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3baaebf4-e07b-46c6-a2af-5bd598770095, pool_id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: d4092a69-3126-47bf-99a5-32b743b7cd67, pool_id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203242518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203254018Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203264018Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203269258Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203300669Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203306399Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203312199Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.203316609Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.214358273Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.214404354Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool)"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:34.284828021Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15859638551243678161"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:34.284920562Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4362447466759465143"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.309968882Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.317878452Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.317955953Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.317968973Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.317989703Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.318005204Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.318159826Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.318173707Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.359395161Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4531ed09-fb4d-493a-ab70-e12857ce0b15/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.367464123Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_4531ed09-fb4d-493a-ab70-e12857ce0b15","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.537493519Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.54546317Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.54549827Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.545524Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.545539281Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.545545831Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.555381174Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.555409624Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.650271941Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.65818443Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.65821432Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.65822458Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.658270821Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.658283701Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.658400823Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.658407803Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d0c143e7-a173-42d2-843a-c5f94a0fd260)","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:34.669902756Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.693444269Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f92a8532-ba0d-4943-9c53-cbf333f3e95b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.701344618Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_f92a8532-ba0d-4943-9c53-cbf333f3e95b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.873414261Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.881171927Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.881208028Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.881215748Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.881222498Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.881247338Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.891459168Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.891493118Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.986474117Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.994816493Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.994874574Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.994891754Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.994905125Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.994917645Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.995058148Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:34.995066928Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.031829443Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_25219abe-0fe9-4521-85ec-ce7d104aa4f8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.039945506Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_25219abe-0fe9-4521-85ec-ce7d104aa4f8","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.213172819Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.22120007Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.221244481Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.221251831Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.221257271Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.221263351Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.231361879Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.231397099Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ba9141ae-afce-40cd-9266-493a40c46e44 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.324361282Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332366523Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332413594Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332423364Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332472275Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332489445Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332596507Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.332606607Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.367790885Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ba9141ae-afce-40cd-9266-493a40c46e44/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.375301557Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_ba9141ae-afce-40cd-9266-493a40c46e44","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.548857116Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.557084311Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.557120651Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.557129281Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.557137891Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.557144822Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.567073016Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.567095386Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.657402213Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.66465958Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664695071Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664704141Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664726901Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664764912Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664878014Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.664891464Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d0c143e7-a173-42d2-843a-c5f94a0fd260)","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:35.675186485Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.700748814Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.708695134Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.879552126Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.887379793Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.887410024Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.887421394Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.887428234Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.887433024Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.897290117Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.897324278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.98908909Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996692733Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996726084Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996733404Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996773765Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996784515Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996912727Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:35.996920807Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.030296383Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0b1838a9-eb04-49b1-87fa-fcb9bd223288/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.037810135Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_0b1838a9-eb04-49b1-87fa-fcb9bd223288","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.207708Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.215382135Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.215442196Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.215453006Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.215462966Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.215483437Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.225467702Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.225501483Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"cb23150e-2260-4362-9623-6f1cf720c4aa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.32267252Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332483122Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332518913Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332528593Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332589414Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332605924Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332706816Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.332727196Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.372274501Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cb23150e-2260-4362-9623-6f1cf720c4aa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.380376513Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_cb23150e-2260-4362-9623-6f1cf720c4aa","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.549862381Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.557549726Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.557586706Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.557597257Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.557605717Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.557610997Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.566861159Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.56689306Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.659857253Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667773002Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667842003Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667855083Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667867214Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667878024Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667987236Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.667999546Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d0c143e7-a173-42d2-843a-c5f94a0fd260)","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:36.678734355Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.702596354Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3709573b-558a-4113-ae7a-6b8dc15392c8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.710505043Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_3709573b-558a-4113-ae7a-6b8dc15392c8","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.873955074Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.881212382Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.881248002Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.881256342Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.881271653Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.881276973Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.891325709Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.8913611Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.989523164Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.99725138Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.9972813Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.99728839Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.997295991Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.997306761Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.997468934Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:36.997478774Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.035778687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_79a573cb-dfa8-4e36-9e7e-3e2b460b68f7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.043879269Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_79a573cb-dfa8-4e36-9e7e-3e2b460b68f7","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.246413717Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.254645131Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.254708463Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.254806694Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.254951167Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c8821193-671b-45a0-9a99-7032783dba68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f06f8e57-0c5b-434e-91ed-491eb8ad51b6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255045538Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255055009Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.25511047Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255342464Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255357084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255367404Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:894","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255372154Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255376354Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255411945Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255421245Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:910"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255469436Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255476796Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.255570358Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"e27853c7-dd63-4e78-abaa-81c0ff0213d7","remote_addr":"[fd00:1122:3344:101::1]:63717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3052584,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.286901278Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.287145532Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:58822"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.287420987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.287430567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.341504627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.341548308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.393366928Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.393405129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.444458466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.444492877Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.444532697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.444573268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.488065182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.488118243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.488126923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.539494306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.542974156Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.547002637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.559772342Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.564067827Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:37.568024707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d0c143e7-a173-42d2-843a-c5f94a0fd260)","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:37.681437149Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:39.056417332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:39.056519874Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:39.056679447Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:39.540458055Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:40.019541911Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:40.019693573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:40.019703543Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 48.303896ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.028366426Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:40.028420067Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:40.032698322Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 124.8825ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.086053789Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 52.804894ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.157664747Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 37.1292ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.163703533Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 62.057196ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.208210325Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 80.050312ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.218552517Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 135.399554ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.325072558Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 147.480333ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.475146774Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 156.099176ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.494896041Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 538.906901ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.518673019Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 360.123154ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.657291734Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 425.007772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:40.672100104Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 1.161177886s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:41.027684221Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 631.378119ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:41.066178107Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 848.175613ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:41.105700891Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 1.292929422s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:41.708529411Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 1.155587475s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:41.964944275Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 1.044236168s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:42.202850445Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:42.984269992Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4289"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:42.984328433Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4299","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:42.990044203Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4378"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 887.783293ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.012694421Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.091037187Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4426"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.091077528Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4453"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.128374513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.128412084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4330"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 1.416602491s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.130510291Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.177656089Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4372"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.177688119Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.197324204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4444"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.197349555Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4486"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 1.10384016s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.257234127Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.267364045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4518"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.267394666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4524"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.354470695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.354521116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Failed to get switch ID from MGS (retrying in 74.180208ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.386484807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 83.276985ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.461822981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 298.817379ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.545949849Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:43.671719358Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2203795520816278611"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:43.671885531Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10437281365267708969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.673937727Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:43979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.67411168Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"c60d2260-6962-4b39-81be-76034d56c03e","remote_addr":"[fd00:1122:3344:101::2]:43979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.845938059Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.877982872Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.878097774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.878394519Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.878405669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.919660094Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"wait for service Some(\"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a\") failed: Property not found. retry in 1.252317587s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:43.92002905Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968132144Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968178925Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968192245Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968212416Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968224536Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968232006Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968242616Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968252897Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968261227Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968269347Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968275917Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968283987Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968290497Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968296917Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968303638Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968317118Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968322058Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968327038Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968331838Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968336658Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968341498Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968346438Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:43.968351148Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:44.286879704Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15859638551243678161"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:44.286986966Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4362447466759465143"} {"msg":"wait for service Some(\"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a\") failed: Property not found. retry in 1.283152589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:44.37034303Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b\") failed: Property not found. retry in 1.250090508s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:44.55647573Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825192447Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_fac57501-4b98-404d-bb24-cd54171c288a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825258209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_b97cd7b5-3290-493c-92cb-a7e936330b5a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825267209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_internal_dns_ce535360-9682-4b2f-a000-efffcbc2a39b"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825283999Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825521983Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.825819669Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c8a44e2f-0af3-4916-887c-8ab54c3e6327","remote_addr":"[fd00:1122:3344:101::1]:58822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8538585,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.827575139Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.873271122Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.91867445Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.971101561Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:45.971112661Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.004439736Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.005279231Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.005297241Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.034215759Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.034369782Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60173"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.034591566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.034603816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.055649736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.055699676Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.072453471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.072492502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.091115689Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.09117402Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.091403984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.091511096Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.103339244Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.135528929Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.137030175Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.137040885Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.166180217Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 212, 121]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.166216998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.166524483Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.166529443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.166754518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.167128514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.189212852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.206913393Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.836065705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:46.836160597Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 50.382257ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:46.95874164Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 129.59151ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:47.017862248Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 100.355143ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:47.180291371Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 442.565361ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:47.292277849Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 817.508016ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:47.744450692Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 866.001811ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:48.571137704Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 1.310652257s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:49.446169955Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 631.38732ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:50.767027129Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b\") failed: Property not found. retry in 1.053253218s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:51.407882616Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.492975277Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.493068669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.493380235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.493657939Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fd66d6cc-938c-42be-8da1-63973f545458","remote_addr":"[fd00:1122:3344:101::1]:60173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6459233,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.495934949Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.531008605Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.531131618Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39220"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:52.539058687Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.5392006Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"68774728-ce82-4820-876e-1eee576a8094","remote_addr":"[fd00:1122:3344:101::1]:39220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7907,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:52.540706346Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.691935083Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:52.692100795Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60974"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:53.674290579Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2203795520816278611"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:53.674390341Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10437281365267708969"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:54.288615581Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15859638551243678161"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:54.288900106Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4362447466759465143"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:55.195817678Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17599775339958856220"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:55.676096755Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12749476767218469494"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:55.82779249Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16521831450609735741"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:56.700276666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:56.700417448Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"75e09baf-bded-4edc-9ec6-6ff13c7e4fa6","remote_addr":"[fd00:1122:3344:101::1]:60974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008237,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:56.702947873Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:56.867702327Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:56.86784088Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35565"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T15:59:56.876036814Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:56.876134405Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"82e2e107-17a1-4813-a670-d5c01c298ec9","remote_addr":"[fd00:1122:3344:101::1]:35565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8244,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T15:59:56.877740954Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:57.01814724Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T15:59:57.018284922Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:63565"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_4261cb69-1241-41ad-8a9d-28acbdefe53b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-11T16:00:01.027881547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.027988989Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"98198406-1348-4c5f-852e-66806ada3195","remote_addr":"[fd00:1122:3344:101::1]:63565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009631,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.03032803Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.024334488Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.0244843Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:58620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.044287858Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"02224d26-f5cb-4db3-9c96-7a76031404ed","remote_addr":"[fd00:1122:3344:101::1]:58620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":19716,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1097084582, ip_addr: 65.100.46.166, stratum: 2, ref_time: 1728662400.8660731, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.046484417Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.046490937Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.075290633Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.075380205Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40910"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.075631309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.075641109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.094723624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.094760935Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.111480719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.111513639Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.128660481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.128692651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.246739055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.246773136Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.353189335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.353220555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.443163795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.443216956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.543568959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.54361192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.659508926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.659548247Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.65970627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.659807421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.691836164Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.691875195Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.691885595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.691897225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.691906265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.691989977Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ef61596b-4986-40ae-bada-ee971352119a","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized)]","latency_us":616498,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.69215786Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"68.032437ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef61596b-4986-40ae-bada-ee971352119a\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef61596b-4986-40ae-bada-ee971352119a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.761298274Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.761519318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.761527728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.779384892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.779438113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.798020049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.79805658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.816046926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.816091647Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.833881519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.83391867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.852028228Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.852060089Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.870107006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.870142176Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.887964389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.88800313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.905576139Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.905616829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.905774092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.905888994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.937949237Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.937985938Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.937994038Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.938001158Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.938015788Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:01.93807392Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1a18d9e5-7f44-473e-a407-9e81e1a178ac","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized)]","latency_us":176673,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:01.938194952Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"133.762762ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a18d9e5-7f44-473e-a407-9e81e1a178ac\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a18d9e5-7f44-473e-a407-9e81e1a178ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.073528839Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.073758803Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.073779264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.092483572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.092529333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.109250407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.109282757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.127619409Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.12766308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.145330961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.145363171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.163096893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.163135103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.182491473Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.182530334Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.200944497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.200981328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.218079178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.218124429Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.218279562Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.218378834Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250175932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250225193Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250236963Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250245544Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250255304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.250350745Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"536701cc-1fd9-4b4b-9325-3a864c049e92","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized)]","latency_us":176713,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.250529418Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"103.510457ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"536701cc-1fd9-4b4b-9325-3a864c049e92\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"536701cc-1fd9-4b4b-9325-3a864c049e92\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.355441561Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.355676576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.355693386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.373744433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.373777014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.392050555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.392082065Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.408523254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.408562755Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.425329829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.42536528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.442088943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.442127114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.458934299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.45896871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.475832986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.475876747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.492802225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.492847495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.493031609Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.49313366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523437393Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523520474Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523529534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523538784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523546334Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.523613476Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"99e1f559-6c88-423c-8355-5507a8af892b","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized)]","latency_us":168061,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.523775618Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"127.255967ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99e1f559-6c88-423c-8355-5507a8af892b\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"99e1f559-6c88-423c-8355-5507a8af892b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.652178114Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.652420749Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.652432619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.669291895Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.669325115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.686478187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.686512267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.703660779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.703694839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.720967943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.720994373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.73843408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.73847194Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.756221722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.756249613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.773068428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.773104129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.790011195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.790048726Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.790203979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.790300011Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.821737493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.821791304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.821803104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.821829985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.821837565Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.821906506Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"578f9ad2-8214-4f3f-b67e-a8489a1a2fcc","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized)]","latency_us":169603,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:02.822029268Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"123.655446ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"578f9ad2-8214-4f3f-b67e-a8489a1a2fcc\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"578f9ad2-8214-4f3f-b67e-a8489a1a2fcc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.94736618Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.947596604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.947611864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.966248182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.966288882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.984234407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:02.984267628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.002374836Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.002415337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.021912459Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.02194201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.03905575Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.039086881Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.056045879Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.0560804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.072970826Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.073007717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.091196117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.091236867Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.09140109Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.091519122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122528367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122564288Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122573168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122580228Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122586528Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.12270333Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"412edb2c-e62e-423d-b270-51e6ff7b25ec","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized)]","latency_us":175227,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.122858073Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"231.828728ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"412edb2c-e62e-423d-b270-51e6ff7b25ec\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"412edb2c-e62e-423d-b270-51e6ff7b25ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.356249343Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.356449846Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.356466966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.37431599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.374360371Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.390091387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.390139148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.407488883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.407531354Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.42325511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.42329117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.440516013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.440544063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.458396787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.458430098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.475486257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.475530068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.493558555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.493622316Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.493774869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.493880391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.526271749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.52631816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.526327981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.52633567Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.526343111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.526420542Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d471c467-1301-4254-8ba0-b5de2d4a2cdb","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized)]","latency_us":170088,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.526574975Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"276.262221ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d471c467-1301-4254-8ba0-b5de2d4a2cdb\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d471c467-1301-4254-8ba0-b5de2d4a2cdb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.803983628Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.804772262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.804791772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.823551291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.823592352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.841426116Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.841464606Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.858506665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.858539806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.876285228Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.876331399Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.894197252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.894231993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.912075567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.912108937Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.929417551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.929470632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.947486159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.94754234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.947709483Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.947828015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979017422Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979073183Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979081104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979087854Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979094414Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:03.979151175Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4000e236-9324-4547-a8a5-3bdf7c195f4d","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized)]","latency_us":175084,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:03.979288117Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"358.216477ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4000e236-9324-4547-a8a5-3bdf7c195f4d\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4000e236-9324-4547-a8a5-3bdf7c195f4d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.337671959Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.337895863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.337910163Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.357630981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.357664391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.376105086Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.376131156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.393741427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.393788387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.41325815Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.413288751Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.431567783Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.431592804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.44786201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.447886491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.465193265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.465227166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.482359828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.482394178Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.482575842Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.482673723Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.512312025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.512338446Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.512346676Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.512354666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.512360776Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.512448398Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6c1a29d8-2834-4710-bffb-7069d663fb39","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized)]","latency_us":175152,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.51258197Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"218.322497ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6c1a29d8-2834-4710-bffb-7069d663fb39\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6c1a29d8-2834-4710-bffb-7069d663fb39\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.731375393Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.731669929Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.731681599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.748875752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.748904992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.766146946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.766178066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.783258887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.783292058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.800860085Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.800896538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.817358318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.817387508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.834970368Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.835004079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.852242792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.852270313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.869843982Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.869873633Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.870046216Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.870154268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901232495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901274216Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901281306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901289316Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901295436Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:04.901383608Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"34b4b2be-e71d-4172-87ca-63e112cf6630","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized)]","latency_us":170340,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.901539491Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"745.735046ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34b4b2be-e71d-4172-87ca-63e112cf6630\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34b4b2be-e71d-4172-87ca-63e112cf6630\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:04.981511999Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"214.361629ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.64687936Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.647128724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.647141234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.667623665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.667645425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.68610249Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.686125281Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.704187039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.70421939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.722515982Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.722546212Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.740424327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.740446707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.758682349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.758707269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.775611257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.775638427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.793470861Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.793508302Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.793694205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.793799517Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.824302025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.824330045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.824338235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.824347036Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.824355426Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:05.824437917Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"87b8205a-1153-45c3-b0fb-736cc1e72a7b","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized)]","latency_us":177959,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:05.82458074Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"451.115321ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87b8205a-1153-45c3-b0fb-736cc1e72a7b\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87b8205a-1153-45c3-b0fb-736cc1e72a7b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.275753177Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.27594853Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.275969961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.293306066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.293342157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.311366814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.311404545Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.328071038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.328102419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.346930001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.346966971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.365196092Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.365236163Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.383994354Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.384024614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.403179112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.403206992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.420290893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.420335844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.420497337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.420596198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453236483Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453287194Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453309695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453319905Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453327295Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:06.453401526Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"0034ac6e-1ec9-4a6e-a17b-1b483e5099ab","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized)]","latency_us":178062,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:06.453543269Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"597.953391ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0034ac6e-1ec9-4a6e-a17b-1b483e5099ab\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0034ac6e-1ec9-4a6e-a17b-1b483e5099ab\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.051308928Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.051675175Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.051693605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.071661216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.071707927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.090325315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.090358506Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.107569708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.107601302Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.125987353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.126020494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.145039569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.14507359Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.162974105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.163008836Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.181266587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.181307048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.198969389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.19900869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.199186023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.199304865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232069852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232103003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232112253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232120743Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232129653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.232217615Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"44bbb2cc-1354-4bf9-87cf-003f5e16e528","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized)]","latency_us":181238,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:07.232345027Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"741.250791ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"44bbb2cc-1354-4bf9-87cf-003f5e16e528\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"44bbb2cc-1354-4bf9-87cf-003f5e16e528\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.972637306Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.972933591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.972942971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.992037888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:07.992071579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.011064803Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.011092274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.031609225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.031645916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.050564909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.050598089Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.069477412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.069510772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.08807191Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.08811425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.107054984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.107088074Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.124590263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.124623343Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.12482316Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.124932659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156302801Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156358972Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156371122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156380603Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156388263Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.156472154Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e531a575-87d7-4b19-a146-e4d8d0782fc6","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized)]","latency_us":184219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.156622277Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"645.809361ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e531a575-87d7-4b19-a146-e4d8d0782fc6\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e531a575-87d7-4b19-a146-e4d8d0782fc6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.80229865Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.802506514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.802514554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.821627671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.821683522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.838364376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.838407907Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.855308774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.855353515Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.873609716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.873647967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.891458941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.891498091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.909960037Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.909992767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.927222211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.927276772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.945011824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.945068385Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.945225318Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.94533269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975574163Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975616583Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975628564Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975637894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975647184Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3312","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:08.975722325Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"64fee024-5ca1-4326-9e61-c2c5a61df1bf","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\", TimeNotSynchronized), (\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\", TimeNotSynchronized), (\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\", TimeNotSynchronized), (\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\", TimeNotSynchronized), (\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\", TimeNotSynchronized)]","latency_us":173813,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:08.975874388Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.161184097s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"64fee024-5ca1-4326-9e61-c2c5a61df1bf\", \"content-length\": \"124\", \"date\": \"Fri, 11 Oct 2024 16:00:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"64fee024-5ca1-4326-9e61-c2c5a61df1bf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.135327071Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.135617546Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.135633117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.156670687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.156721688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.174170036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.174204346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.19203729Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.192084141Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.208943068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.208976509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.227057977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.227092148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.243868103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.243906374Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.262341199Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.262370859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.281411485Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.281446545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.281648159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.281760681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.36403429Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.364090761Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.364096661Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.364102291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.364107301Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.453040278Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.456958837Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.461240072Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.465232152Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.46963954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.474526336Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.479228559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.48384486Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.488229798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:10.492679996Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:12.9254866Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:12.925652703Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"acf8c23e-d913-4f2a-8b74-b2b39f609802","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51439","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"200"} {"msg":"Profile for oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.485654167Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.485764709Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.5352231Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.535374663Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.544260659Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.544373581Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.571080462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.571191764Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.592648522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:13.592788834Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 64.386195ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:13.635776591Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 46.046121ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:13.680193574Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 33.385998ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:13.688800995Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 86.372856ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:13.737671866Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 139.587416ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:14.016223553Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 71.108867ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:14.747511654Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 98.550041ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:14.863788332Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 260.367511ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.220490225Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 285.296131ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.337179192Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 117.845573ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.598661976Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 262.494698ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.643743281Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 71.566983ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.693578039Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 425.720775ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.756548317Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 246.745228ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.78734219Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 196.820855ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.813645203Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 136.358869ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.844203241Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 255.549752ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:15.927950397Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 255.894275ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.207875598Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.06677727s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.244815227Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 495.089581ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.267662639Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 485.199503ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.308605558Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.090908492s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.345400645Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 487.196489ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.472575689Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 656.754764ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.774468324Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 989.217337ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.807141978Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 671.597377ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:16.970326045Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 654.977945ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.321011684Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 508.958404ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.440290238Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 612.436862ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.447756239Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 871.531597ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.649991311Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 859.037696ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.804585311Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.342609221s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.957770946Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.111960123s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:17.985254778Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.244949955s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:18.070365055Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.232065446s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:18.530828596Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 622.584861ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:18.673238326Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 716.946296ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.108351244Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 1.353302282s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.306540852Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.188468191s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.314269918Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.183406223s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.324759484Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.281275716s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.771793831Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.129784852s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:19.833911818Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:20.203737798Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"690.918147ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 779.571246ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:20.513041361Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.318177158s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:20.520246588Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 1.356196912s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:20.668157996Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.255976752s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:20.973036281Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.328656736s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:21.062864829Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 693.419648ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:21.302627262Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 703.868127ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:21.851873052Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 696.288256ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.005829257Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 1.085265623s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.033356981Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.339910712s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.237948375Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 916.459473ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.402203671Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 694.153466ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.567309032Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 542.180278ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:22.710300354Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 755.668622ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.128448461Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.000049042s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.261536949Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.117863806s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.270463086Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.413327864s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.327358925Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 785.077111ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.587130699Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 816.944459ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:23.893748155Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.261157821s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:24.271662515Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 645.486007ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:24.381574496Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 1.358517509s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:24.396314445Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 1.256978735s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:24.719407792Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.380780028s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:24.749814096Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.218522683s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:25.036718247Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.30741891s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:25.542120076Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 991.409802ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:25.76266167Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 847.770224ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:25.984397866Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754\") failed: Property not found. retry in 1.219954848s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:26.140813533Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa\") failed: Property not found. retry in 1.107401788s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:26.263938406Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338\") failed: Property not found. retry in 967.91337ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:26.76352652Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819\") failed: Property not found. retry in 1.041528993s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:26.844447876Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd\") failed: Property not found. retry in 1.220238985s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:26.860108017Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101296689Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_4b839812-7fb8-4fcb-a634-cb341d2563aa"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.10136923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_9294d795-2faf-4441-98ad-2dd811e7f6fd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101375491Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_25cc7ecc-cb7c-422f-963f-3d85eb1cf819"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101380711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_4fa56da7-438c-473d-9129-08746ad3a338"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101385351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_cockroachdb_1cb86e06-0936-41b8-be70-17c6f64c6754"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101390311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.101857429Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.102222225Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"27626e8c-6d9d-4a0e-9b66-e630c70ca0f7","remote_addr":"[fd00:1122:3344:101::1]:40910","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":17981692,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.13604409Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53283"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:28.165992845Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3740"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:41.398401185Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.641903905Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.642461984Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"e06485f2-7cd1-453d-92fc-d9d0c3b042fe","remote_addr":"[fd00:1122:3344:101::1]:53283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":17506042,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.645104331Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51566"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.647104946Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"15bc9d61-0e4d-44ad-82e4-cf042ffd93da","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51566","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1051,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:45.655925661Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"767.163455ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: ce535360-9682-4b2f-a000-efffcbc2a39b (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b97cd7b5-3290-493c-92cb-a7e936330b5a (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fac57501-4b98-404d-bb24-cd54171c288a (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cca02c4d-2be9-4dd4-9596-4e0a36360393, kind: Service { id: 4261cb69-1241-41ad-8a9d-28acbdefe53b }, name: Name(\"ntp-4261cb69-1241-41ad-8a9d-28acbdefe53b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 212, 121])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1cb86e06-0936-41b8-be70-17c6f64c6754 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fa56da7-438c-473d-9129-08746ad3a338 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b839812-7fb8-4fcb-a634-cb341d2563aa (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9294d795-2faf-4441-98ad-2dd811e7f6fd (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 4f86de3f-8d13-490d-a006-d09a3379f9ed, kind: Service { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 }, name: Name(\"external-dns-10233eca-efbb-4a96-b4d0-64ff838602d1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 28])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 3514f1d1-fa4c-4978-bf6b-62ba5e4eb9b2, kind: Service { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b }, name: Name(\"external-dns-f39e5250-aaac-40a9-aaa6-a51c0ca4870b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 3ca38684-22f6-459d-b91b-a7379a06940a, kind: Service { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a }, name: Name(\"nexus-1310d80c-fc44-47fa-8bcf-d31cf0f9747a\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 241])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d55024f5-93bb-43a7-a789-d4ba224bad0e (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 47a5923b-767b-4b6b-8cb1-2df97fa7ef00, kind: Service { id: d55024f5-93bb-43a7-a789-d4ba224bad0e }, name: Name(\"nexus-d55024f5-93bb-43a7-a789-d4ba224bad0e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 10])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 6775aa22-8df6-40af-b153-e564ea876260, kind: Service { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 }, name: Name(\"nexus-eae7ca62-f742-444e-8e53-ebcdd5555c15\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 162])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 20f4616a-d168-4d06-a4ad-b77acda6e19d (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8d391c0-2fb1-4100-9118-512649dc74b8 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 86737208-a70a-460a-ad16-ad713c02c908 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: e4bef209-d9cb-4903-9b52-5db55c7a2614 (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 93d9ec30-4959-4c0e-bb2c-51d9c0973d52 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9cfbc7a4-6651-4c50-afa4-351965f3dc4f (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External } } } }, OmicronZoneConfig { id: d0103a3d-1e9d-489c-92e8-c62281a92162 (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9e7ab8cf-edc5-4d65-be1d-c93badf19ed4 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 13c8a497-1377-4b6d-b254-3a0ec484b220 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: b5d7217c-e5ff-436e-933c-76945a12cc60 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External } } } }, OmicronZoneConfig { id: 91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 1351ea4e-78d7-4c66-ba8b-696a64af4684 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6d251c32-24cb-476d-8b9c-fd35c69d2c9c (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.688775588Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.689314947Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51150"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.690421787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.690429437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: ce535360-9682-4b2f-a000-efffcbc2a39b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.717730696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.717776997Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b97cd7b5-3290-493c-92cb-a7e936330b5a, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.737916801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.737955912Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fac57501-4b98-404d-bb24-cd54171c288a, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.755716864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.755753714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1cb86e06-0936-41b8-be70-17c6f64c6754, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.77433452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.774378851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4fa56da7-438c-473d-9129-08746ad3a338, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.792295996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.792343967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4b839812-7fb8-4fcb-a634-cb341d2563aa, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.810906423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.810944324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9294d795-2faf-4441-98ad-2dd811e7f6fd, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.832289109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.832339439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 25cc7ecc-cb7c-422f-963f-3d85eb1cf819, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10233eca-efbb-4a96-b4d0-64ff838602d1, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.851345173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.851382534Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 10233eca-efbb-4a96-b4d0-64ff838602d1, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.934334791Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:45.934381592Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 20f4616a-d168-4d06-a4ad-b77acda6e19d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.020047597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.020105178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 20f4616a-d168-4d06-a4ad-b77acda6e19d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93d9ec30-4959-4c0e-bb2c-51d9c0973d52, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.090523315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.090573316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 93d9ec30-4959-4c0e-bb2c-51d9c0973d52, dataset_name: DatasetName { pool_name: ZpoolName { id: 4531ed09-fb4d-493a-ab70-e12857ce0b15 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cfbc7a4-6651-4c50-afa4-351965f3dc4f, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.168883962Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.168928382Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9cfbc7a4-6651-4c50-afa4-351965f3dc4f, dataset_name: DatasetName { pool_name: ZpoolName { id: f92a8532-ba0d-4943-9c53-cbf333f3e95b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d0103a3d-1e9d-489c-92e8-c62281a92162, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.241052009Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.241186722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d0103a3d-1e9d-489c-92e8-c62281a92162, dataset_name: DatasetName { pool_name: ZpoolName { id: 25219abe-0fe9-4521-85ec-ce7d104aa4f8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e7ab8cf-edc5-4d65-be1d-c93badf19ed4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.323575879Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.32362451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9e7ab8cf-edc5-4d65-be1d-c93badf19ed4, dataset_name: DatasetName { pool_name: ZpoolName { id: ba9141ae-afce-40cd-9266-493a40c46e44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 13c8a497-1377-4b6d-b254-3a0ec484b220, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.407519724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.407573465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 13c8a497-1377-4b6d-b254-3a0ec484b220, dataset_name: DatasetName { pool_name: ZpoolName { id: a5ef1ef0-586b-48ab-b6cd-2fa12f8464e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5d7217c-e5ff-436e-933c-76945a12cc60, dataset_name: DatasetName { pool_name: ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.482549082Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.482607093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b5d7217c-e5ff-436e-933c-76945a12cc60, dataset_name: DatasetName { pool_name: ZpoolName { id: 0b1838a9-eb04-49b1-87fa-fcb9bd223288 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4, dataset_name: DatasetName { pool_name: ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.55021083Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.550260991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4, dataset_name: DatasetName { pool_name: ZpoolName { id: cb23150e-2260-4362-9623-6f1cf720c4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1351ea4e-78d7-4c66-ba8b-696a64af4684, dataset_name: DatasetName { pool_name: ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.614903847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.614958908Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1351ea4e-78d7-4c66-ba8b-696a64af4684, dataset_name: DatasetName { pool_name: ZpoolName { id: 3709573b-558a-4113-ae7a-6b8dc15392c8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d251c32-24cb-476d-8b9c-fd35c69d2c9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.682364992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.682426003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6d251c32-24cb-476d-8b9c-fd35c69d2c9c, dataset_name: DatasetName { pool_name: ZpoolName { id: 79a573cb-dfa8-4e36-9e7e-3e2b460b68f7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.749008252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.749075153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.749337447Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.7494811Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.903437094Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.903701419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.90371645Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.90372238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.9037294Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.904988792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.906478408Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.93108711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.931322675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.942069513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:46.943676712Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.12801079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128098111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128102921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128106581Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128109991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128115101Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128119041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128124172Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.128221323Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.133594888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.138928471Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.14454607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.151206977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.157473697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.322382434Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.326948944Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.331430333Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.335358302Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.339960859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.344687386Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.349332607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.354671801Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.359547497Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.364700967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.369833037Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.375932014Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.381612364Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.387889504Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.394573722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.401477793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.406654154Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.41157006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.417449914Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.423073763Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.430257779Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.436556739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.525207577Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.606072287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.681624182Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.750904771Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.81403217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.825674935Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.825702985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.896369916Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 142, 28]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.896399077Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.900606151Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.900622181Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.969308678Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 196, 201]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.969336188Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.971543234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:47.971557616Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.03260515Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 220, 10]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.03263499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.034225418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.034240888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.097659383Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 208, 162]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.097694423Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.099613307Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.099627077Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.22961227Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 130, 241]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.229663571Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.24952079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249551841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249697333Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249709263Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249821965Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249832445Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249846026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.249853406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.253851586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.253868396Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.258532698Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.260763148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.317448933Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.323135533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.323157184Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.324497157Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.327927528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.327979578Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.327989869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.422113322Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.577156406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.586165704Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.594754485Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.602810626Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.612559928Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.621328291Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.631697954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.640662811Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.649238522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:48.658118518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.302189079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.302595836Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.309847543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.310003656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.31934441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.319529033Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.473741812Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.473951156Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.474143579Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.482606028Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.482804631Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.503485045Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.715930806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.717392912Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.794156361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.795171918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.795186789Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.838140603Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.885887162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.904919126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.90568925Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:50.910987553Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:51.044564669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:51.044995547Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 63.671992ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.097168823Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 40.976162ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.111570426Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 52.783513ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.124242359Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 61.064974ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.138207374Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 135.662317ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.167916736Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 57.77202ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.182092294Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 35.28806ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.222166589Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 78.422355ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.243413812Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 54.404156ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.261553171Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 230.555252ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.276017805Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 57.008034ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.291395928Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 73.114371ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.304976564Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 33.904666ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.406004948Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 125.133918ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.522051577Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 27.214692ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.615517018Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:51.616235869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 130.183502ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.694082588Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 280.959732ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.773584665Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 71.354903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.824083122Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 51.863915ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.891268992Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 521.967555ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:51.977556618Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 147.051937ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.037178805Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 28.182986ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.077959811Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 67.829023ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.144554071Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 249.162863ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.296053162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 73.75317ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.348490414Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 106.564608ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.394870958Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 54.540692ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.493000132Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:52.493073843Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 512.258031ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.825493403Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 248.308086ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.887261518Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 135.059175ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:52.962756594Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 287.207806ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.020923546Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 607.083593ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.060208926Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 285.715805ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.087351723Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 102.298119ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.171611703Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 108.640702ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.213069402Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 350.907114ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.261468132Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 137.032ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.28640655Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 447.790677ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.305857661Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 232.922007ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.328901076Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 688.618147ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.34392652Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 181.217177ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.457127619Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 304.980712ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.477765101Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 370.737863ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.489481557Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 134.48629ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.50388333Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 231.532691ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.516914779Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 233.221539ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.530783743Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 59.581579ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.542074671Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 515.884234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.553612134Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 326.435634ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.576207351Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 51.935953ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.613544706Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.185010556s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.626233189Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 320.326085ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.654118489Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 317.617644ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.667708398Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 162.325372ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.677725444Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 561.603128ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.689063993Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 270.862922ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.76519273Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 673.86467ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.777203981Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 401.176884ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.788849446Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 526.473799ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.801885905Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 561.308445ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.85681811Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 945.175941ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.871141882Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 627.624226ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.919164605Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 455.574715ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:53.990377616Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 591.50134ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.004484334Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 570.945533ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.049598957Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 635.184838ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.062737247Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.311476673s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.083791457Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.463110392s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.209050068Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 652.941122ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.269843906Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 587.940689ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.34236424Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 781.154532ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.432045395Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 759.983586ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.482515871Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 560.956048ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.499386958Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:54.522556515Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:54.52284675Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 510.975856ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.564919679Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 612.524208ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.613153217Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 637.692974ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.63897793Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.116824312s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.732537223Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 74.779323ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.821649398Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 974.205648ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.840603111Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 514.885ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.85819493Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 85.990127ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.912099187Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.349573028s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:54.962985811Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.480798531s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.044465093Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 184.749896ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.088380574Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 969.434582ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.125642439Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 908.309887ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.15647841Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.278182672s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.235295125Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 575.115164ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.256691461Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.286012062s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.275775176Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 553.806412ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.298078748Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.482924633s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.315997423Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.20833937s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.390499621Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 596.721454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.411035672Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 729.004553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.830677104Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.28639053s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.847590441Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 957.954628ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.863023362Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 962.984987ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.892057242Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 993.36579ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:55.908401019Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.201170378s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.051532944Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 848.943618ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.077423208Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 768.97394ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.108358442Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.119155029s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.328693912Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 781.219288ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.527406003Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:56.529041632Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 776.829864ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.543625758Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:00:56.543921133Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.477650224s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.574072503Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 725.160023ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.58700708Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 800.444179ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.614213418Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 64.65032ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.75269843Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 739.380857ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.813112882Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 146.428772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.829725294Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 800.015492ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.841349418Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.271911002s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.868705038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 631.766752ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.889399982Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 523.842133ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.916760292Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 700.081642ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.938307461Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 209.378653ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:56.989892717Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.147031836s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.147374964Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 208.109856ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.217855222Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.214286795s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.268498271Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.322755373s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.324403154Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.260032128s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.336581447Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.482521262s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.348223202Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.443922731s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.427327452Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 679.345258ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.440133357Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 792.174095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.455980695Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 960.549997ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.46821736Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.020669188s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.534292801Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.275254326s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.569730763Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.222736771s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.651725344Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.382425776s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:57.664170802Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.087785155s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.067020329Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 1.196175087s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.134302441Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.107962795s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.153147452Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 703.722652ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.259658733Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.373572578s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.306413835Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 908.436578ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.440700904Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 639.72658ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.495827682Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 745.990305ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.567024093Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 563.000382ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.61130054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.027988965s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.659436966Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.040454127s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.845132048Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.38716291s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.858625535Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 999.887209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.883433721Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 544.499596ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.89476771Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 534.831589ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:58.974862977Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 931.863907ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.058248902Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.016691089s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.148689671Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 932.395886ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.16628522Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 899.922031ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.18392983Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 693.383638ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.2738667Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 887.655263ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.325317284Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 1.194488368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.341499718Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 982.879553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.359517755Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 552.813981ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.450502673Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 939.88048ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.521992069Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 942.460111ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.694500579Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.109819718s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.706186424Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 904.927967ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.89551488Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 568.485425ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.907794656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 553.169613ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:00:59.993922739Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.175387846s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.006996769Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.181119875s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.017205178Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 799.816211ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.133619723Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 666.858889ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.146327616Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.47834031s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.207502791Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.379299934s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.224928027Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 805.514268ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.258902114Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 637.659407ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.406405535Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.101297903s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.475782184Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 855.401354ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.488905114Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 784.767807ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.549785364Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 731.840235ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.57689485Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.262978635s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.871958963Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 677.940045ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.884454832Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 725.131357ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.897083554Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.135382368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.908888571Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.90939158Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.909422771Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.909573223Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.909698295Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.916817681Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:00.923299825Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.177115789s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:00.947057412Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.469398846s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.056751449Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.481122081s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.075260054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.053659074s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.193797726Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 762.703228ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.210347167Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 60.110901ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.288085923Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 53.230253ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.297366066Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 72.963586ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.305893275Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.48551678s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.319682398Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 871.882622ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.345680025Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.301706324s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.355318354Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 144.216195ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.363820693Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 93.790369ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.372423604Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 55.44561ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.389071317Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 237.229848ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.45702213Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.692594819Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.114430899s"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.000395362s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.707013842Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.265580761s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.718805089Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 129.235184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.735731046Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 254.661024ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.751504934Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 416.117107ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.764708286Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 816.15733ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.775410014Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 728.674172ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.78603095Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.461270356s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.796093167Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 491.218662ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.877979115Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 749.036257ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:01.987995288Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 300.288632ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.018592105Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.200371657s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.055929711Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.02503903s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.136361364Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.359679394s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.146882969Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 561.756012ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.194236621Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 844.301406ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.229956088Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.364650794s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.260270211Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 498.087612ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.330079417Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 636.435512ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.381173435Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 720.316931ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.524574474Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 733.73201ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.534738353Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.311296686s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.573260389Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 650.64686ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.602785248Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.450524296s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.667006856Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.327572544s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.71736131Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.050165419s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.745731449Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 690.560199ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.765511446Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 689.643211ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.814573068Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.435762348s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.837208016Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 821.357217ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:02.993123075Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.286961121s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.028084229Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 529.89588ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.083861359Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.205425744s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.170645863Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 515.52945ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.25530798Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.336965019s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.263541205Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.435540617s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.270233763Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.122336312s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.276376101Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 808.466954ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.283264082Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 699.055744ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.465228088Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.127496804s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.514638086Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 811.291631ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.523036103Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 510.976799ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.625701627Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.401686503s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.634724926Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 592.101159ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.780527587Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.263474439s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.805032467Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.032727172s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.877973769Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.214260491s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:03.894684812Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 774.490702ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.055051469Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.326314359s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.103164975Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.351839165s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.128576041Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 904.044283ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.146494466Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 1.242184404s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.172998321Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.493071236s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.28277233Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 981.18735ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.324166157Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.035460998s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.343399235Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.28129493s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.38183403Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.197646016s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.38982227Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.456628224s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.407783716Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.040760835s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.610698741Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.427494878s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.654547911Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 553.336216ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.714452772Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.281503377s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.839409927Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 947.300362ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:04.919997033Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 551.009949ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.046538116Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 888.175219ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.061474648Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 512.394764ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.078510458Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 874.273934ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.118553621Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.442006073s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.278559562Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 523.736706ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.314256599Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 957.844043ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.388757178Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 1.033631743s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.423536289Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.287656777s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.438142445Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.415441032s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.490620907Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.141383966s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.598109055Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 917.966607ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.607735955Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 967.735846ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.615676784Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.199291353s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.661495729Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 512.178951ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.672742377Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.292800784s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.785936205Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:05.818815442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:05.819063107Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 614.877981ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.848772329Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.412417458s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.875260394Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 647.141936ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.883455798Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 716.525447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:05.96102831Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 926.838789ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.003954235Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 60.792578ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.024869242Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.293981909s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.09135396Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 76.793358ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.098887272Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.344218671s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.142968357Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 277.437907ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.189743878Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.417735712s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.209822251Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.101561514s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.358400571Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 971.811217ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.466804065Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.156467544s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.475199803Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 334.980024ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.4835671Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.099010423s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.53536566Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 972.743977ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.542658278Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 657.61968ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.591009887Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 700.090853ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.687213287Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.148138009s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.730200382Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.356135544s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.738221862Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 751.179257ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.749679934Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 588.181664ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.827867337Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.351933347s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.871058986Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 909.995333ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.915015478Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.432801403s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:06.940018558Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.213067735s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.089072486Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.37731233s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.259455099Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 973.947903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.297522038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 558.504984ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.394040883Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 602.17401ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.402679255Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 1.354878324s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.425887443Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 610.242313ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.448720014Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 556.628353ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.469064571Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.449766652s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.497072233Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 564.225978ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.510089782Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.256035679s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.523340905Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 558.939444ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.636171327Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.48869518s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.643520516Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 981.529077ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.65059967Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.316420023s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.835461177Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 999.776602ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.887089364Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.256850785s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:07.962112652Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 1.389255246s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.017148759Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 961.298218ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.035758046Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 1.496262704s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.069609951Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.35793652s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.082156741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.040190203s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.102499639Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 713.228507ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.204169194Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 525.642619ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.232852448Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.23029498s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.280602167Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.188437247s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.310300599Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 824.426672ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.383462764Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 772.563028ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.641435696Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.411751007s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.649867974Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.324281515s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.76922061Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.38023266s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.792450938Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 523.647091ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.804980578Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 554.347982ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.898507741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 794.586998ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.930165227Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 947.894039ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:08.96161552Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 612.384645ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.00888742Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 638.779454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.145591482Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 905.180828ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.155466565Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.184536677s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.165200236Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 936.197491ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.220206132Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.250091939s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.230248219Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 872.720509ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.342545761Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 1.239385941s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.416755635Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 648.945903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.42843637Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.350707597s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.452595545Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.449974136s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.464139847Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 923.015342ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.510581443Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.039580061s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.523236056Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 948.805891ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.578043628Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.273126984s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.632905752Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 737.804016ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.737991698Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.072983827s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.797606535Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 839.961541ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:09.919617149Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.027388589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.073763907Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.156971411s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.085267049Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.245442914s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.095539949Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 852.630708ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.107136163Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 732.523368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.171668276Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 757.250353ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.189103713Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 634.969701ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.23107912Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 593.868471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.361946719Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 839.224242ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.44793591Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 990.47615ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.49067039Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 912.8458ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.507812421Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 939.210115ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.540574117Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.338490054s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.575211085Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 971.94422ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.666514049Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.467222876s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.773163683Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.385721892s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.816770869Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 798.776857ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.87774653Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 698.563333ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.888322506Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 633.131836ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.919986302Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.353628707s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.934832633Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 520.228561ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.949999789Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 561.105359ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.964660807Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.392530217s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.981167877Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 831.316062ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:10.99902345Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 1.19678594s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.116433553Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.295510829s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.252471253Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 885.390616ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.299151093Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.267710443s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.355290989Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.403449576s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.438274166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 925.175825ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.484662292Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 840.64028ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.499068614Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.349906751s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.512959858Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 513.454011ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.541136004Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 700.00689ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.569612934Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.143326485s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.599153963Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 650.382455ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.650753039Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 1.345916607s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.687996193Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.455004823s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.843079777Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 530.124333ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:11.92639289Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684\") failed: Property not found. retry in 1.391557713s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.067428698Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 905.913282ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.199002089Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.490144423s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.214906409Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 789.772282ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.252599281Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614\") failed: Property not found. retry in 1.128744318s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.284262767Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.314656368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.300094285Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 1.483286686s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.312408021Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 966.752469ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.32426515Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 828.133842ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.354371269Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 1.238052136s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.388118942Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.196140555s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.424114314Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.155172686s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.470945246Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.046238443s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.560143893Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 1.439945821s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.639766882Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.484329438s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.755369113Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 805.083009ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.854921462Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.309350808s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:12.875950281Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 1.494621446s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.05067446Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.256944101s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.061424219Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 702.490476ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.11841806Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 682.370386ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.197388955Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 820.139962ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.30626041Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.464447978s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.318498675Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 508.429129ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.621721592Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 1.457518148s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.633520739Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 898.359556ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.644350959Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 522.846056ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.654564649Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.23367741s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.674165923Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 862.344079ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.687217892Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.014753548s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.715960486Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 875.610593ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.809850536Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.382951984s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.836704028Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 707.485027ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:13.894795558Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60\") failed: Property not found. retry in 811.666091ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.094196781Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 694.574228ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.139201842Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.031119564s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.149275418Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 751.146022ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.188715151Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 1.275287409s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.198369041Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 919.3911ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.252067294Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 1.029923876s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.332458056Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908\") failed: Property not found. retry in 1.15687379s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.552612424Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 1.028941025s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.561335447Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c\") failed: Property not found. retry in 1.403833509s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.569512991Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 795.014132ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.611316515Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 554.140658ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.696298898Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162\") failed: Property not found. retry in 1.463088115s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.741811047Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220\") failed: Property not found. retry in 1.413355037s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.792830673Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8\") failed: Property not found. retry in 944.809693ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.844863567Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.201943248s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.929741458Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b\") failed: Property not found. retry in 703.213334ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:14.949655568Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f\") failed: Property not found. retry in 768.412353ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.101824372Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 715.704905ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.185813947Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4\") failed: Property not found. retry in 1.412166911s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.197241378Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 1.05296104s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.230624374Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 787.320782ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.259081794Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 525.973209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.374189066Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 793.523572ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.416021781Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a\") failed: Property not found. retry in 851.973654ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.485369429Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 1.000435879s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.599887101Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:15.727244048Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:15.74384611Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e53b9537-5ce9-49ce-8ff5-156746743e4b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39993","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5548,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4\") failed: Property not found. retry in 537.328907ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.909948498Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e\") failed: Property not found. retry in 1.01132499s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:15.918530359Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1\") failed: Property not found. retry in 884.297451ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:16.058782672Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52\") failed: Property not found. retry in 1.243367208s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:16.16510265Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a\") failed: Property not found. retry in 1.426196022s","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:16.229848517Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15\") failed: Property not found. retry in 783.288769ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:16.304342426Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 543.07412ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:16.618628057Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d\") failed: Property not found. retry in 947.656496ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:17.181374503Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.16661376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_clickhouse_20f4616a-d168-4d06-a4ad-b77acda6e19d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166793683Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_9cfbc7a4-6651-4c50-afa4-351965f3dc4f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166801523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_9e7ab8cf-edc5-4d65-be1d-c93badf19ed4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166807854Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_1351ea4e-78d7-4c66-ba8b-696a64af4684"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166828044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_d55024f5-93bb-43a7-a789-d4ba224bad0e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166834244Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_eae7ca62-f742-444e-8e53-ebcdd5555c15"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166839884Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_nexus_1310d80c-fc44-47fa-8bcf-d31cf0f9747a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166845564Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_b5d7217c-e5ff-436e-933c-76945a12cc60"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166851304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_external_dns_10233eca-efbb-4a96-b4d0-64ff838602d1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166857015Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_external_dns_f39e5250-aaac-40a9-aaa6-a51c0ca4870b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166862725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_oximeter_9b75dfd3-0808-4303-9b81-ee0a6e0cfa9a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166868515Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_86737208-a70a-460a-ad16-ad713c02c908"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166874095Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_e4bef209-d9cb-4903-9b52-5db55c7a2614"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166879695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_d0103a3d-1e9d-489c-92e8-c62281a92162"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166885505Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_13c8a497-1377-4b6d-b254-3a0ec484b220"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166890835Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_pantry_d8d391c0-2fb1-4100-9118-512649dc74b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166896405Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_93d9ec30-4959-4c0e-bb2c-51d9c0973d52"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166901475Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_91d7ec3c-dd8b-4266-bcdc-cafd528ebfd4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.166906655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3308","zone":"oxz_crucible_6d251c32-24cb-476d-8b9c-fd35c69d2c9c"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.169269607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.17284294Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.196124659Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7ede58b6-ba5a-4765-b8bf-4b21ede47425","remote_addr":"[fd00:1122:3344:101::1]:51150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32506081,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.208347223Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.209540744Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.209731778Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.44496175Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.446849923Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.500536876Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.517979903Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.654924728Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.745342457Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.825851411Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:18.873709482Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"3.913420522s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:18.95617808Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:19.081692505Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:19.389137716Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:19.710595302Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:19.928050592Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:20.460511536Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:20.727915213Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:21.039176581Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:21.916983921Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:22.824159147Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"5.183386871s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:23.18751507Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:24.515452607Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:26.163575819Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:27.416380866Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-11T16:01:28.040309046Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"21.633493957s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/d30c9cf6-9f94-4f76-affa-4751a592517c/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:28.735980867Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69ca0f4d-e2a2-4b54-a4b5-136ad76bfc7b\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:31 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"d30c9cf6-9f94-4f76-affa-4751a592517c\\\"\", request_id: \"69ca0f4d-e2a2-4b54-a4b5-136ad76bfc7b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:31.234949425Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e044d39-d0bf-4a46-985e-d64004ba46aa\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:34 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"0e044d39-d0bf-4a46-985e-d64004ba46aa\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:35.093911413Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"02a9c1f9-6974-4013-a7b0-91b794859e6b\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:39 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"02a9c1f9-6974-4013-a7b0-91b794859e6b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:39.552622848Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b9f538f-e9f7-49a6-853d-9ac53ac8365b\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"8b9f538f-e9f7-49a6-853d-9ac53ac8365b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:45.005914383Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:45.779257778Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:45.779328059Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b74aa307-55d3-425d-aec7-a2ba1e31a2df","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35746","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":27,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66cbae9b-fe00-4773-bd47-8fb80aa94327\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"66cbae9b-fe00-4773-bd47-8fb80aa94327\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:52.143787591Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f066741e-a8fb-477d-8fa7-ef3bc32dee4e\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:01:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"f066741e-a8fb-477d-8fa7-ef3bc32dee4e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:01:55.69306208Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32c8f16b-5704-49b9-bd98-d752f0611814\", \"content-length\": \"173\", \"date\": \"Fri, 11 Oct 2024 16:02:00 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"d0c143e7-a173-42d2-843a-c5f94a0fd260\\\"\", request_id: \"32c8f16b-5704-49b9-bd98-d752f0611814\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:00.458120134Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:03.904510644Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:03.997302815Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36968"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.002356583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.19915276Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"f797502a-8c4c-4bfb-8010-95515dd9188d","remote_addr":"[fd00:1122:3344:101::a]:36968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":201393,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.201496922Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.201693065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.20880041Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:04.209009903Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"4f4f65cb-ccc5-45a8-9a2e-861002e28339","remote_addr":"[fd00:1122:3344:101::a]:36968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9279,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.586685808Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43604"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.596874097Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3514f1d1-fa4c-4978-bf6b-62ba5e4eb9b2, kind: Service { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b }, name: Name(\"external-dns-f39e5250-aaac-40a9-aaa6-a51c0ca4870b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4f86de3f-8d13-490d-a006-d09a3379f9ed, kind: Service { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 }, name: Name(\"external-dns-10233eca-efbb-4a96-b4d0-64ff838602d1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 28])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3ca38684-22f6-459d-b91b-a7379a06940a, kind: Service { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a }, name: Name(\"nexus-1310d80c-fc44-47fa-8bcf-d31cf0f9747a\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 241])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 47a5923b-767b-4b6b-8cb1-2df97fa7ef00, kind: Service { id: d55024f5-93bb-43a7-a789-d4ba224bad0e }, name: Name(\"nexus-d55024f5-93bb-43a7-a789-d4ba224bad0e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 10])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6775aa22-8df6-40af-b153-e564ea876260, kind: Service { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 }, name: Name(\"nexus-eae7ca62-f742-444e-8e53-ebcdd5555c15\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 162])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597459167Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597500418Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597535409Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597564759Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597571299Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.597588129Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.59760918Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"59b9c8e5-36b8-4b60-b0d6-3129e1ad521f","remote_addr":"[fd00:1122:3344:101::a]:43604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":10008,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:10.599404892Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.319663264Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34413"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.319797416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.545652784Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"fa775c64-4776-4f66-8eb7-84b9c67e4926","remote_addr":"[fd00:1122:3344:101::a]:34413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":225849,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.545782686Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57935"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.545850998Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62101"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546207514Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3514f1d1-fa4c-4978-bf6b-62ba5e4eb9b2, kind: Service { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b }, name: Name(\"external-dns-f39e5250-aaac-40a9-aaa6-a51c0ca4870b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4f86de3f-8d13-490d-a006-d09a3379f9ed, kind: Service { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 }, name: Name(\"external-dns-10233eca-efbb-4a96-b4d0-64ff838602d1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 28])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3ca38684-22f6-459d-b91b-a7379a06940a, kind: Service { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a }, name: Name(\"nexus-1310d80c-fc44-47fa-8bcf-d31cf0f9747a\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 241])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 47a5923b-767b-4b6b-8cb1-2df97fa7ef00, kind: Service { id: d55024f5-93bb-43a7-a789-d4ba224bad0e }, name: Name(\"nexus-d55024f5-93bb-43a7-a789-d4ba224bad0e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 10])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6775aa22-8df6-40af-b153-e564ea876260, kind: Service { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 }, name: Name(\"nexus-eae7ca62-f742-444e-8e53-ebcdd5555c15\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 162])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546257415Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546317946Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546334856Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546359176Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546377317Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546396837Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546415328Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"13257ef2-a2b2-4fa6-a608-8ffab232ed75","remote_addr":"[fd00:1122:3344:101::a]:62101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":443,"response_code":"204"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.546920586Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/c8821193-671b-45a0-9a99-7032783dba68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.54714395Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547154961Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3514f1d1-fa4c-4978-bf6b-62ba5e4eb9b2, kind: Service { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b }, name: Name(\"external-dns-f39e5250-aaac-40a9-aaa6-a51c0ca4870b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4f86de3f-8d13-490d-a006-d09a3379f9ed, kind: Service { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 }, name: Name(\"external-dns-10233eca-efbb-4a96-b4d0-64ff838602d1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 28])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3ca38684-22f6-459d-b91b-a7379a06940a, kind: Service { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a }, name: Name(\"nexus-1310d80c-fc44-47fa-8bcf-d31cf0f9747a\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 241])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 47a5923b-767b-4b6b-8cb1-2df97fa7ef00, kind: Service { id: d55024f5-93bb-43a7-a789-d4ba224bad0e }, name: Name(\"nexus-d55024f5-93bb-43a7-a789-d4ba224bad0e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 10])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6775aa22-8df6-40af-b153-e564ea876260, kind: Service { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 }, name: Name(\"nexus-eae7ca62-f742-444e-8e53-ebcdd5555c15\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 162])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547179981Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547212671Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547238112Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547246682Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547252972Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547259212Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547266122Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"403854f5-5557-43f2-822c-5bd0227fce9e","remote_addr":"[fd00:1122:3344:101::c]:57935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":566,"response_code":"204"} {"msg":"Reading ledger from /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547439155Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.547647619Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"GET","req_id":"2e736e5e-dc51-415c-a289-9cee05400a46","remote_addr":"[fd00:1122:3344:101::a]:34413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1607,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.792717964Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54729"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.792968638Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3514f1d1-fa4c-4978-bf6b-62ba5e4eb9b2, kind: Service { id: f39e5250-aaac-40a9-aaa6-a51c0ca4870b }, name: Name(\"external-dns-f39e5250-aaac-40a9-aaa6-a51c0ca4870b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 201])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4f86de3f-8d13-490d-a006-d09a3379f9ed, kind: Service { id: 10233eca-efbb-4a96-b4d0-64ff838602d1 }, name: Name(\"external-dns-10233eca-efbb-4a96-b4d0-64ff838602d1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 142, 28])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3ca38684-22f6-459d-b91b-a7379a06940a, kind: Service { id: 1310d80c-fc44-47fa-8bcf-d31cf0f9747a }, name: Name(\"nexus-1310d80c-fc44-47fa-8bcf-d31cf0f9747a\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 241])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 47a5923b-767b-4b6b-8cb1-2df97fa7ef00, kind: Service { id: d55024f5-93bb-43a7-a789-d4ba224bad0e }, name: Name(\"nexus-d55024f5-93bb-43a7-a789-d4ba224bad0e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 220, 10])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6775aa22-8df6-40af-b153-e564ea876260, kind: Service { id: eae7ca62-f742-444e-8e53-ebcdd5555c15 }, name: Name(\"nexus-eae7ca62-f742-444e-8e53-ebcdd5555c15\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 162])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.793024059Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.79305764Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.79307461Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.793097851Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.793111301Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.793117051Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:11.793131521Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9ee62694-0ace-49f0-9a14-92d7131bcb82","remote_addr":"[fd00:1122:3344:101::b]:54729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:15.812183442Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:15.812274423Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"962b9c09-11aa-4d3b-affa-467cac451852","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48355","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:19.836823942Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:50908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:20.006269348Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"db4dab79-2028-4286-82db-66c62ef592d7","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130943,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:21.91679596Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:21.925473133Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6262b9cd-a2a3-4c89-98db-026b2a4f8c7f","remote_addr":"[fd00:1122:3344:101::a]:55369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8570,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:22.224314832Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:22.224573977Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"85a41df1-2971-403a-8a37-aa042a6f76ec","remote_addr":"[fd00:1122:3344:101::a]:55369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5663,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:22.341750425Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c48a8059-c442-4ddd-a84a-48c8a5edcc63","remote_addr":"[fd00:1122:3344:101::a]:55369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":13204,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:30.904834639Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:30.91290426Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"da41fdb0-2cf0-444c-9b21-59709d020580","remote_addr":"[fd00:1122:3344:101::a]:60719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8015,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:30.919441865Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:30.919537727Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c2fbe360-a481-4d16-814a-30ac7345227c","remote_addr":"[fd00:1122:3344:101::a]:60396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.123545241Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.123580461Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6a82b416-b0fe-4761-9939-084bc58b6ac9","remote_addr":"[fd00:1122:3344:101::a]:60396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.133411794Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d6cd1b33-aab7-49a2-afb3-66751afe94bb","remote_addr":"[fd00:1122:3344:101::a]:60396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.223480806Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34464"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.229100795Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.347131148Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.433704339Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fc2ee101-8685-4c3e-82cf-d49ba4f8d09b","remote_addr":"[fd00:1122:3344:101::a]:34464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":210136,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.440132382Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8e6c98c1-4c91-448f-b0bf-4eb3241da28f","remote_addr":"[fd00:1122:3344:101::a]:34464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6248,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.503364723Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/c8821193-671b-45a0-9a99-7032783dba68/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.503874682Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f06f8e57-0c5b-434e-91ed-491eb8ad51b6/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.504763908Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.506143482Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.506333605Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"83563010-fef9-4d9c-9d58-5ddf206f6afb","remote_addr":"[fd00:1122:3344:101::a]:34464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":16506,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.604137593Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54052"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.605605909Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b21fc0de-1d6f-4505-b1ba-e88162314636","remote_addr":"[fd00:1122:3344:101::b]:54052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.827907024Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.828058696Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4bf9bd59-e0b5-4ef6-bcf4-6ee88e8f954d","remote_addr":"[fd00:1122:3344:101::c]:33361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.883301357Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.883525061Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"9f94adea-ded3-4c67-9b08-8038697ccd33","remote_addr":"[fd00:1122:3344:101::b]:56710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.891345988Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.891372979Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4a9f9af-c7d1-45f6-8923-29fc2c580129","remote_addr":"[fd00:1122:3344:101::b]:54052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:31.907669235Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"19f9a802-2e91-4883-a69e-ae9ac53e241b","remote_addr":"[fd00:1122:3344:101::b]:54052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":221,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.040821024Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.040853025Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"feb3584a-dfd2-4301-a84a-9f6e92f5c649","remote_addr":"[fd00:1122:3344:101::c]:33361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":749,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.11115546Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.111353683Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1134dea6-4f1f-41ba-a3ed-1e664d0c4bbb","remote_addr":"[fd00:1122:3344:101::c]:51926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.121903389Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4bd4847d-268d-45a6-872a-efaeadbf9b01","remote_addr":"[fd00:1122:3344:101::c]:33361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.142182915Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48087"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.142298937Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.181866822Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.241555391Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c4d5c442-68f8-4db2-afb7-79b0a0a89640","remote_addr":"[fd00:1122:3344:101::b]:48087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99287,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.241741144Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45197"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.242443826Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.246375778Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cd0e12b4-18a1-4710-976a-084ba7ea7811","remote_addr":"[fd00:1122:3344:101::b]:48087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4594,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.281220387Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.337809911Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f9fa2161-51af-4758-b629-b8380dad1eca","remote_addr":"[fd00:1122:3344:101::c]:45197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95416,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:32.338150827Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"43722067-79e8-475d-af95-be238fc8ee32","remote_addr":"[fd00:1122:3344:101::c]:45197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:45.847221575Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:45.847315787Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"86c8eeec-3042-45c2-b8ac-6d0463d92e2c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62640","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:02:49.844044655Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"02091520-44d1-498e-8113-f1ea5da1765d","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7665,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.769365565Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.769561078Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ce266531-754e-4c22-b5e9-1c393eca3f9e","remote_addr":"[fd00:1122:3344:101::a]:48382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.818896075Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.818929845Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"00460228-9158-4325-95b0-70d7f6284c5c","remote_addr":"[fd00:1122:3344:101::a]:48382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.904867435Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.905056109Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1ec216d8-5521-42ef-8427-89f3aab8249a","remote_addr":"[fd00:1122:3344:101::a]:57581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.963553316Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38557"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:00.963690879Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.013259749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.073897115Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"712a371f-6228-4ef5-bda9-c3359cdb0687","remote_addr":"[fd00:1122:3344:101::a]:38557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110271,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.074409153Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"24b66cd2-c5a2-483d-b4fe-4b3b04bc4944","remote_addr":"[fd00:1122:3344:101::a]:38557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":250,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.074424844Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"de934556-25ab-490f-8ec3-9b25114db1ae","remote_addr":"[fd00:1122:3344:101::a]:48382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.540302228Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.54044944Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"12d9745c-5b32-4e2a-a6bd-bcbe13c064bb","remote_addr":"[fd00:1122:3344:101::b]:42761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.593115345Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.593149386Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"100a7bab-f848-4eb6-a370-18a8f02a8bdc","remote_addr":"[fd00:1122:3344:101::b]:42761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.62564737Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.625837769Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a77d2303-ed55-45fa-88f8-444fb2ccf118","remote_addr":"[fd00:1122:3344:101::b]:36380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.75111642Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50059"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.751241653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.788636759Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.846487496Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fd391ded-85a5-4a08-8e54-d26cb97cb068","remote_addr":"[fd00:1122:3344:101::b]:50059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95297,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.846664199Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48089"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.84672031Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55687"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.847071766Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"19690ada-b5d2-408e-8b5a-e33022e789c6","remote_addr":"[fd00:1122:3344:101::b]:50059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":278,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.847095376Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"144e2ced-4ec7-4eba-905a-f37d8563a33d","remote_addr":"[fd00:1122:3344:101::c]:55687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":303,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.848110834Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9d3f9f93-b3a1-4d35-950d-f43cf45aed4f","remote_addr":"[fd00:1122:3344:101::c]:48089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":977,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.848321518Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d1da298c-93ab-4a77-a1aa-3aef0a64c16e","remote_addr":"[fd00:1122:3344:101::b]:42761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1485,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.864239007Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.864267058Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c337707d-a4ff-4732-88c8-c0dcd8857f5f","remote_addr":"[fd00:1122:3344:101::c]:48089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.889114015Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44942"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.889235497Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.929211809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.98793058Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9cbb667e-6e73-41c4-a7ce-981978147f16","remote_addr":"[fd00:1122:3344:101::c]:44942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98736,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.989105861Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"69dafbdb-9fc1-4148-a2dc-3920d767ac44","remote_addr":"[fd00:1122:3344:101::c]:48089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1039,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:01.989250914Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b2068a93-4622-4424-88e3-e8090103d329","remote_addr":"[fd00:1122:3344:101::c]:44942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:15.880967144Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:15.881053766Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"c83c632f-822a-4c7b-8a07-49ee158da6dd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34704","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:19.845204542Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"137aa95a-2a8e-44c8-bf19-3df4298d1d91","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8446,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:30.898430259Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:30.899280844Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a95b1632-ed18-48f5-9949-c8a9156e068a","remote_addr":"[fd00:1122:3344:101::a]:35867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":792,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:30.953046259Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:30.95310124Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8eeca6e2-bd13-4602-a311-83bce091c152","remote_addr":"[fd00:1122:3344:101::a]:35867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.06581545Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46546"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.066009233Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"04d0824f-09e7-438b-8234-29090f370649","remote_addr":"[fd00:1122:3344:101::a]:46546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.072903254Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"476d0d7a-ab53-4075-8e0c-92720e18c8b0","remote_addr":"[fd00:1122:3344:101::a]:35867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":168,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.221105978Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41851"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.221203179Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.261217432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.317496911Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9e9c9e60-e1bc-42d9-9083-22a174a133bd","remote_addr":"[fd00:1122:3344:101::a]:41851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96334,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.318388077Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"78a9ec4e-f80d-4852-8b04-99c378aaea1f","remote_addr":"[fd00:1122:3344:101::a]:41851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":713,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.600084555Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.60263544Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"97270a4f-27c5-4afc-9ee1-de541cb2a256","remote_addr":"[fd00:1122:3344:101::b]:52666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2424,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.679869327Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.679908377Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4f4fd844-8a24-4779-8226-b1f7d69e1ff7","remote_addr":"[fd00:1122:3344:101::b]:52666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.696087802Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b9b733d9-d9da-49a8-8708-ea97839a19f2","remote_addr":"[fd00:1122:3344:101::b]:52666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1422,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.795219043Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.795421567Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"266f9497-46b0-4561-a686-db0ae84aa534","remote_addr":"[fd00:1122:3344:101::b]:43406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.894738791Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54805"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.895745009Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:31.935510617Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.000169373Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"31f2c825-9972-4955-9d3d-4956df2ff44d","remote_addr":"[fd00:1122:3344:101::b]:54805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105353,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.000424178Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63635"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.000748714Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fce383ac-678a-4886-8525-b05a806210c8","remote_addr":"[fd00:1122:3344:101::c]:63635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.000797674Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7a805dbc-ed39-42a8-bc8e-8d8cde37a011","remote_addr":"[fd00:1122:3344:101::b]:54805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":303,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.034359674Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.034556777Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"856646a7-6f7a-4bac-b44e-55cdb26acc5a","remote_addr":"[fd00:1122:3344:101::c]:61248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.048994771Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.049017611Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"de0090e7-78e7-4c10-8556-15bfc4afc3bf","remote_addr":"[fd00:1122:3344:101::c]:63635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.058910245Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e5a2cd14-4a51-4d82-8f7f-4d41c946920b","remote_addr":"[fd00:1122:3344:101::c]:63635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.1394544Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41775"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.139568022Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.180835567Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.234090133Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"623283ae-9e09-441d-96fa-8fefa22dcce9","remote_addr":"[fd00:1122:3344:101::c]:41775","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94574,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:32.234906907Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f521d980-2461-4907-8c89-abc3ba6e8cd8","remote_addr":"[fd00:1122:3344:101::c]:41775","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":655,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:45.913725269Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37388"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:45.91381206Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e1e25de4-b8c7-4d4d-be6a-93043699be4f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37388","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:03:49.844468087Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"f9749d7b-cb29-4997-b687-01a9a7e7f540","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8063,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.797128487Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.797245779Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"00e1f6e0-cb9f-4a02-8bad-74cf84ff96d7","remote_addr":"[fd00:1122:3344:101::a]:52384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.903592037Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44852"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.90374856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.941713117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.998031196Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ecb676bf-24d3-4d17-b571-15b086dd8504","remote_addr":"[fd00:1122:3344:101::a]:44852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94335,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.998437163Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.998495754Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"11bce7ac-363c-41d4-8810-77081cc465fa","remote_addr":"[fd00:1122:3344:101::a]:52384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:00.998561005Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3804c5d5-cf18-4eb8-937e-5ccc6b09c2e7","remote_addr":"[fd00:1122:3344:101::a]:44852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":255,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.009220953Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0c89c765-2fdb-436f-96a4-87a037be9488","remote_addr":"[fd00:1122:3344:101::a]:52384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1004,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.03409811Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.034249242Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"88fa4ec0-079c-47c1-94bd-78a28eca63f5","remote_addr":"[fd00:1122:3344:101::a]:46650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.600308906Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.60166461Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"31c47078-732f-4e92-a724-1cf9caa96713","remote_addr":"[fd00:1122:3344:101::b]:55130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1304,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.623562965Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.623584445Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"724d4ece-7ea6-4961-aaec-0bb430b4759e","remote_addr":"[fd00:1122:3344:101::b]:55130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.640184277Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e1e5d730-c51c-445c-9648-1897bc9b1a73","remote_addr":"[fd00:1122:3344:101::b]:55130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.71896303Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42530"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.719148104Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"633c6b0e-7c5d-4ff9-b16b-f76bcbf6f40f","remote_addr":"[fd00:1122:3344:101::b]:42530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.721479675Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36221"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.721594657Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.764539881Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.821723576Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0c071500-3db0-4455-8667-cb57f2edfb60","remote_addr":"[fd00:1122:3344:101::b]:36221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100163,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.821835388Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62376"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.822708483Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8ef14555-65f6-468d-a0c3-a777e3b5e85e","remote_addr":"[fd00:1122:3344:101::c]:62376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":802,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.822852066Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"738e2cb8-093c-45b4-b00b-c0a272583f0b","remote_addr":"[fd00:1122:3344:101::b]:36221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.846403429Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.84643451Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f55d5619-46ad-4132-8d34-37b852416202","remote_addr":"[fd00:1122:3344:101::c]:62376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.856286393Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a4a99f9f-1ed4-4859-ab9e-6a63d4b40e9d","remote_addr":"[fd00:1122:3344:101::c]:62376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.995151822Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:01.995350156Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ce6deeae-c6a1-43b4-a450-e9862312335c","remote_addr":"[fd00:1122:3344:101::c]:38129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:02.026721507Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46621"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:02.026807929Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:02.067275599Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:02.128409863Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7b55aeae-a7d5-475f-a2ab-a8e2c8d23140","remote_addr":"[fd00:1122:3344:101::c]:46621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101637,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:02.128729989Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"da3eae70-2972-4c38-a135-b77e0de567e2","remote_addr":"[fd00:1122:3344:101::c]:46621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:15.949542293Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:15.950336227Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"c9f7502c-ccf9-4cc8-beba-159ee4d1ae99","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35629","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":701,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:19.862798816Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"f0966a3e-1568-4bc2-8d8e-7748834e6e85","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5778,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:30.935392503Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:30.935560437Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a8069265-4531-4855-9528-828565a0163d","remote_addr":"[fd00:1122:3344:101::a]:56637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.003867066Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.003917797Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"97ba8c07-073c-4b65-80b4-e1e9fab71148","remote_addr":"[fd00:1122:3344:101::a]:56637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.015546402Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b25be215-a469-4bf7-925f-c884c1f17d5f","remote_addr":"[fd00:1122:3344:101::a]:56637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":283,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.177176561Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.177329314Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"23aa4c1d-e975-40ea-bd06-6a72b0b77f08","remote_addr":"[fd00:1122:3344:101::a]:59591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.404369932Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55079"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.404568236Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.45149271Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.520882699Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"4a7b0daa-bbc6-4453-98cf-32753c62bc11","remote_addr":"[fd00:1122:3344:101::a]:55079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116378,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.521247045Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a2fcbcab-3c4b-4736-b4c3-6036a1a4a7a3","remote_addr":"[fd00:1122:3344:101::a]:55079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.675510685Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.676658475Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0eee2386-bcc2-4f77-8f9f-3c86514006cb","remote_addr":"[fd00:1122:3344:101::b]:42958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1067,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.719780243Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.719815814Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e098657e-b493-4611-a7f5-a5d37cb12905","remote_addr":"[fd00:1122:3344:101::b]:42958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1355,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.735639621Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0ae0d816-f3b1-4e76-aea9-5ad5654dab52","remote_addr":"[fd00:1122:3344:101::b]:42958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":320,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.790047167Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.79019294Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"16515a0b-fbf3-43b9-8b04-8cbaa1d180a1","remote_addr":"[fd00:1122:3344:101::c]:44733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.853890789Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.853978659Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"26d33cf0-68f2-4c53-94fa-fdc6d56fdad5","remote_addr":"[fd00:1122:3344:101::c]:44733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.866329637Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fc796225-6712-4d7b-9485-1a84aaf7d21a","remote_addr":"[fd00:1122:3344:101::c]:44733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.866394658Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38668"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.86651546Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.913423304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.97351865Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"738b0d18-318d-46bc-8b9d-cb178890fc5d","remote_addr":"[fd00:1122:3344:101::b]:38668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107035,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.973694363Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.973974008Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3634171f-d87b-48e4-8b29-11586aa83f8a","remote_addr":"[fd00:1122:3344:101::b]:38668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:31.9740948Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"62cd284a-2d4d-431d-855b-2912fd3e7015","remote_addr":"[fd00:1122:3344:101::b]:33347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":226,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.068392147Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.068610811Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fc3c9b91-b3b4-4e21-bb1a-d9820dc51d87","remote_addr":"[fd00:1122:3344:101::c]:40158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.321156227Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62055"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.322007102Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.360863855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.413346837Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3f095d96-ec57-4086-9fa4-b8c0692a118b","remote_addr":"[fd00:1122:3344:101::c]:62055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92084,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:32.413652112Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e3f75725-d2a7-4145-b3a6-936978c04709","remote_addr":"[fd00:1122:3344:101::c]:62055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:36.987411658Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:45.984387723Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63709"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:45.984480774Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"d9798ec0-c9d1-479a-8522-76784cb4f31e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63709","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:04:49.940813234Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"4aa6c815-fbb7-4b4a-a844-b25873d81337","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8244,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.769537697Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50678"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.76969109Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"916d1c88-85b9-4573-9291-d9ffdc47dc72","remote_addr":"[fd00:1122:3344:101::a]:50678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.897378673Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.897588076Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5627f1c7-5395-4e19-8168-e1fd46eeffa9","remote_addr":"[fd00:1122:3344:101::a]:53271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.940526331Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.940584642Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7d8861e8-8d1e-4e93-8b73-3d69088490b9","remote_addr":"[fd00:1122:3344:101::a]:50678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.944162835Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61519"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.944830796Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:00.981546661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.035472016Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b1175c2c-f17a-427c-ac20-83c445b7bf47","remote_addr":"[fd00:1122:3344:101::a]:61519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91235,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.035976896Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d02d32da-9769-4ed4-b360-d9757a2bfd2e","remote_addr":"[fd00:1122:3344:101::a]:50678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":335,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.035991035Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"33ed6213-e51e-4bc7-9ae1-0a136011fb4f","remote_addr":"[fd00:1122:3344:101::a]:61519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":278,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.54444058Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.544554442Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b50b0cfc-cee5-46f4-b885-c6dea6902fc3","remote_addr":"[fd00:1122:3344:101::b]:53470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.58716129Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.587196671Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8182f7cf-b2b5-4eaa-a3a6-aa843b8fcb41","remote_addr":"[fd00:1122:3344:101::b]:53470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":912,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.598479579Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"71b36855-bf5e-4205-9096-2082d3529897","remote_addr":"[fd00:1122:3344:101::b]:53470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1499,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.65430148Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48082"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.654514843Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c1115194-b1b0-4c68-89e2-71d90215f1b5","remote_addr":"[fd00:1122:3344:101::b]:48082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.666517034Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54589"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.666603646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.705329816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.763792443Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1a1997c6-05fb-4d3c-9a96-7bea4aba6236","remote_addr":"[fd00:1122:3344:101::b]:54589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97220,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.76478572Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fec6534c-f91f-4808-9665-e6f85b00466d","remote_addr":"[fd00:1122:3344:101::b]:54589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":792,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.820234895Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.820442978Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2f7cc97f-064f-4389-9712-b2a0840b10c9","remote_addr":"[fd00:1122:3344:101::c]:46708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.822433693Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.822534565Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3033640d-50f5-4be3-9d16-9be6ef6c3490","remote_addr":"[fd00:1122:3344:101::c]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.836920088Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.836938328Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d149f698-584e-40f8-85be-bf07bc8197b9","remote_addr":"[fd00:1122:3344:101::c]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.845349756Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b7022fe4-76b6-46e1-94b1-a0180c95701e","remote_addr":"[fd00:1122:3344:101::c]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":258,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.880013305Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35228"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.880148817Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.916542376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.970213169Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"436b130b-5507-48bc-b2e3-698e464dfab1","remote_addr":"[fd00:1122:3344:101::c]:35228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:01.970569575Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b8514928-e9cb-43c9-9509-937c6d0f6fc7","remote_addr":"[fd00:1122:3344:101::c]:35228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:16.018996208Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:16.019082969Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"7333227a-a030-4ec3-a13c-c6fa5723afb6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49351","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:19.941077215Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"d4f73332-ff7a-4126-9f90-151dcb32b0e0","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.006357543Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.006540117Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6ed305f3-1471-4542-90eb-da6c4f6cb19c","remote_addr":"[fd00:1122:3344:101::a]:42476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.05399512Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.054023481Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"397534cd-d201-4d70-aff7-9b7564583416","remote_addr":"[fd00:1122:3344:101::a]:42476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":662,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.063325704Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"700ad7cd-d4a9-4c62-9688-8466c2dc6ce9","remote_addr":"[fd00:1122:3344:101::a]:42476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.109556047Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.110282009Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"687d98f1-60ec-4cb6-ac9c-66f79b6ccad9","remote_addr":"[fd00:1122:3344:101::a]:33474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":677,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.224312942Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65094"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.224404944Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.262503563Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.317734284Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"003c0081-904f-40a0-b11a-bf11f558ee5a","remote_addr":"[fd00:1122:3344:101::a]:65094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93359,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.317999268Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"84afd8ab-f363-4c6c-8c01-ed7130153350","remote_addr":"[fd00:1122:3344:101::a]:65094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.708724242Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.708978476Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"36a2ff88-72c6-48bd-a926-7262940e6f9b","remote_addr":"[fd00:1122:3344:101::b]:40235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.808489984Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.808736559Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"e19fa115-de5e-479f-a5ec-73fb2698a45f","remote_addr":"[fd00:1122:3344:101::b]:35493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.826229546Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.826270467Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ce67f4b2-71da-46dc-a698-90914b760a48","remote_addr":"[fd00:1122:3344:101::b]:40235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.839161043Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"23ee76a4-613c-4a04-8705-d333fe5523aa","remote_addr":"[fd00:1122:3344:101::b]:40235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1351,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.924261918Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.9243751Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3ea3a844-113d-45ca-94fc-e0e55685a85f","remote_addr":"[fd00:1122:3344:101::c]:39735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.965762867Z","hostname":"buskin","pid":1572,"sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1184","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.965791368Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ab953311-4046-4b9a-acb4-dfb0402da4eb","remote_addr":"[fd00:1122:3344:101::c]:39735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:31.974819126Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"599fb0a8-74c3-467a-9aab-0d565dce05ea","remote_addr":"[fd00:1122:3344:101::c]:39735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":351,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.122193275Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39524"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.122963999Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.159241576Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.213406077Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8de12798-de22-4d48-aafc-ed8f0a2bca17","remote_addr":"[fd00:1122:3344:101::b]:39524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.21352414Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.213799004Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c3b7fded-3ad5-4ca7-b6bc-d7115656d370","remote_addr":"[fd00:1122:3344:101::c]:62712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.213903876Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"97a19509-6f5e-457c-a843-93b29dc7d911","remote_addr":"[fd00:1122:3344:101::b]:39524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":220,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.272124769Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58493"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.272274952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4116"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.307640483Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.358887193Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"eda2ba31-b34e-490f-883c-566de210ddb1","remote_addr":"[fd00:1122:3344:101::c]:58493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86681,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:32.359161218Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fe54bacf-7d66-4d1a-8b99-c2294b1d2b1d","remote_addr":"[fd00:1122:3344:101::c]:58493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:46.053836836Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:46.053977899Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1e74306f-c30b-4a0d-aa93-b126339a1174","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46049","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-11T16:05:49.988670824Z","hostname":"buskin","pid":1572,"uri":"/d0c143e7-a173-42d2-843a-c5f94a0fd260","method":"GET","req_id":"d610d2a8-77d7-48ea-978c-cb6299697ba8","remote_addr":"[fd00:1122:3344:101::d]:50908","local_addr":"[fd00:1122:3344:101::1]:47594","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"d0c143e7-a173-42d2-843a-c5f94a0fd260","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8371,"response_code":"200"}