Max level shown:
12024-09-18T18:27:05.649ZINFOphd-runner: phd_runner file = phd-tests/runner/src/main.rs line = 35 runner_args = ProcessArgs { command: Run(RunOptions { propolis_server_cmd: "/work/phd-test/propolis-server", base_propolis_branch: Some("master"), base_propolis_commit: None, base_propolis_cmd: None, crucible_downstairs_cmd: None, crucible_downstairs_commit: Some(Auto), tmp_directory: "/tmp/propolis-phd", artifact_directory: Some("/phd-artifacts"), server_logging_mode: TmpFile, default_guest_cpus: 2, default_guest_memory_mib: 512, artifact_toml_path: "/work/phd-test/artifacts.toml", default_guest_artifact: "alpine", default_bootrom_artifact: "ovmf", include_filter: ["phd_tests::migrate::from_base"], exclude_filter: [], max_buildomat_wait_secs: 1200 }), disable_ansi: false, emit_bunyan: true } target = phd_runner
22024-09-18T18:27:05.651ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 1352, tv_nsec: 172361286 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
32024-09-18T18:27:05.651ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
42024-09-18T18:27:05.651ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = 9d82a99058e7f4d76ccc2141dcd78e7d4730cc78 file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
52024-09-18T18:27:05.652ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/9d82a99058e7f4d76ccc2141dcd78e7d4730cc78/crucible-nightly.sha256.txt
62024-09-18T18:27:05.785ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
72024-09-18T18:27:06.239ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/a9c5fdac442fa6fd5a65e413ea874fe3c62c3fd9/propolis-server.sha256.txt
82024-09-18T18:27:06.355ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
92024-09-18T18:27:06.355ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
102024-09-18T18:27:06.356ZINFOphd-runner: Running 3 test(s) file = phd-tests/runner/src/execute.rs line = 82 target = phd_runner::execute
112024-09-18T18:27:06.356ZINFOphd-runner: Starting test phd_tests::migrate::from_base::migration_from_base_and_back file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
122024-09-18T18:27:06.356ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
132024-09-18T18:27:06.356ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
142024-09-18T18:27:06.356ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
152024-09-18T18:27:06.356ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
162024-09-18T18:27:06.356ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
172024-09-18T18:27:07.012ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
182024-09-18T18:27:07.012ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 618 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store timeout = 600s
192024-09-18T18:27:35.033ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
202024-09-18T18:27:35.033ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
212024-09-18T18:27:35.056ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 2.00M 1.80T 1.91M /phd-artifacts\n target = phd_framework::zfs
222024-09-18T18:27:35.057ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-454ca2f4-334e-48a4-bdc0-a0eefcaf1c52"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = snapshot
232024-09-18T18:27:35.108ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-454ca2f4-334e-48a4-bdc0-a0eefcaf1c52", "phd-artifacts/phd-clone-6c097792-b380-45c4-99e6-f9ccff525ba7"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = clone
242024-09-18T18:27:35.128ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-6c097792-b380-45c4-99e6-f9ccff525ba7"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
252024-09-18T18:27:35.138ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-6c097792-b380-45c4-99e6-f9ccff525ba7 0B 1.80T 48.9M /phd-artifacts/phd-clone-6c097792-b380-45c4-99e6-f9ccff525ba7\n target = phd_framework::zfs
262024-09-18T18:27:35.139ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
272024-09-18T18:27:35.139ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back
282024-09-18T18:27:35.139ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server.tar.gz path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
292024-09-18T18:27:35.139ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/a9c5fdac442fa6fd5a65e413ea874fe3c62c3fd9/propolis-server.tar.gz
302024-09-18T18:27:59.397ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store untar_path = propolis-server
312024-09-18T18:27:59.397ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 518 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
322024-09-18T18:28:12.434ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base_and_back.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server" target = phd_framework::test_vm::server
332024-09-18T18:28:12.434ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/migration_from_base_and_back.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_and_back.stdout.log" target = phd_framework::server_log_mode
342024-09-18T18:28:12.484ZINFOphd-runner: [NEW - EVENT] Launched server with pid 865 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
352024-09-18T18:28:12.484ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 37345 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
362024-09-18T18:28:12.485ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
372024-09-18T18:28:12.485ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
382024-09-18T18:28:12.887ZINFOphd-runner: [SERIAL CONSOLE TASK - START] file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
392024-09-18T18:28:12.889ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] Started instance file = phd-tests/framework/src/test_vm/mod.rs instance_description.instance = Instance { disks: [], nics: [], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Pheidippides-managed VM", id: 1ac4585f-0fce-463f-be7c-77197a03c024, image_id: 00000000-0000-0000-0000-000000000000, memory: 512, metadata: InstanceMetadata { project_id: 2a6c8901-3efd-46b0-99c4-cca58d3ce5a6, silo_id: e4b3d3de-6ed8-4575-a530-40565465357e, sled_id: e19ea39c-5220-45af-b402-b29a50b4b191, sled_model: "pheidippes", sled_revision: 1, sled_serial: "e19ea39c-5220-45af-b402-b29a50b4b191" }, name: "phd-vm-1ac4585f-0fce-463f-be7c-77197a03c024", vcpus: 2 }, state: Creating } line = 428 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
402024-09-18T18:28:12.889ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 404 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
412024-09-18T18:28:12.889ZINFOphd-runner: [PUT_INSTANCE_STATE - START] file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
422024-09-18T18:28:12.889ZINFOphd-runner: [PUT_INSTANCE_STATE - EVENT] Requesting instance state change file = phd-tests/framework/src/test_vm/mod.rs line = 510 path = phd_tests::migrate::from_base::migration_from_base_and_back state = Run target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
432024-09-18T18:28:12.890ZINFOphd-runner: [PUT_INSTANCE_STATE - END] elapsed_milliseconds = 0 file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
442024-09-18T18:28:12.890ZINFOphd-runner: [WAIT_TO_BOOT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
452024-09-18T18:28:12.890ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] waiting for guest to boot file = phd-tests/framework/src/test_vm/mod.rs line = 711 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = 300s vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
462024-09-18T18:28:12.890ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::migration_from_base_and_back step = WaitFor("localhost login: ") target = phd_framework::test_vm
472024-09-18T18:28:12.890ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
482024-09-18T18:28:12.890ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
492024-09-18T18:28:17.131ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 4241 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
502024-09-18T18:28:17.131ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::migration_from_base_and_back step = WriteStr("root") target = phd_framework::test_vm
512024-09-18T18:28:17.131ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::migration_from_base_and_back step = WaitFor("localhost:~#") target = phd_framework::test_vm
522024-09-18T18:28:17.131ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
532024-09-18T18:28:17.131ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
542024-09-18T18:28:17.149ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 18 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
552024-09-18T18:28:17.149ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] Guest has booted file = phd-tests/framework/src/test_vm/mod.rs line = 759 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
562024-09-18T18:28:17.150ZINFOphd-runner: [WAIT_TO_BOOT - END] elapsed_milliseconds = 4259 file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
572024-09-18T18:28:17.150ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
582024-09-18T18:28:17.150ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
592024-09-18T18:28:17.160ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
602024-09-18T18:28:17.161ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
612024-09-18T18:28:17.161ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
622024-09-18T18:28:17.182ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 21 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
632024-09-18T18:28:17.183ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
642024-09-18T18:28:17.183ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
652024-09-18T18:28:17.193ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
662024-09-18T18:28:17.193ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
672024-09-18T18:28:17.193ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
682024-09-18T18:28:17.226ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 32 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
692024-09-18T18:28:17.227ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
702024-09-18T18:28:17.227ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
712024-09-18T18:28:17.237ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
722024-09-18T18:28:17.237ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
732024-09-18T18:28:17.237ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
742024-09-18T18:28:17.259ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 21 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
752024-09-18T18:28:17.259ZINFOphd-runner: [TEST - EVENT] migrating to new Propolis artifact for lifecycle test file = phd-tests/framework/src/lifecycle.rs line = 74 path = phd_tests::migrate::from_base::migration_from_base_and_back propolis_artifact = __DEFAULT_PROPOLIS target = phd_framework::lifecycle vm_name = migration_from_base_and_back
762024-09-18T18:28:17.259ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
772024-09-18T18:28:17.259ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__DEFAULT_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back_lifecycle_0
782024-09-18T18:28:17.259ZDEBGphd-runner: [NEW - EVENT] Examining locally-sourced artifact file = phd-tests/framework/src/artifacts/store.rs line = 56 path = /work/phd-test/propolis-server sha256 = None target = phd_framework::artifacts::store
792024-09-18T18:28:17.259ZDEBGphd-runner: [NEW - EVENT] Locally-sourced artifact is valid, caching its path file = phd-tests/framework/src/artifacts/store.rs line = 69 path = /work/phd-test/propolis-server target = phd_framework::artifacts::store
802024-09-18T18:28:17.260ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base_and_back_lifecycle_0.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9002 server_path = "/work/phd-test/propolis-server" target = phd_framework::test_vm::server
812024-09-18T18:28:17.260ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/migration_from_base_and_back_lifecycle_0.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_and_back_lifecycle_0.stdout.log" target = phd_framework::server_log_mode
822024-09-18T18:28:17.279ZINFOphd-runner: [NEW - EVENT] Launched server with pid 866 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
832024-09-18T18:28:17.279ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 20 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
842024-09-18T18:28:17.280ZINFOphd-runner: [MIGRATE_FROM - START] file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a
852024-09-18T18:28:17.280ZINFOphd-runner: [MIGRATE_FROM - EVENT] Migrating from source at address 127.0.0.1:9000 file = phd-tests/framework/src/test_vm/mod.rs line = 572 migration_id = c8dd732f-1c6c-453f-9d93-64f26ab69d04 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a timeout_duration = 45s
862024-09-18T18:28:17.280ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a vm = migration_from_base_and_back_lifecycle_0 vm_id = 80db91b4-36f2-4be4-9022-5422e94c246a
872024-09-18T18:28:17.280ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9002/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a vm = migration_from_base_and_back_lifecycle_0 vm_id = 80db91b4-36f2-4be4-9022-5422e94c246a
882024-09-18T18:28:17.852ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "466c2a3b-14c6-4b8c-8502-45906d6ff259", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:17 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "466c2a3b-14c6-4b8c-8502-45906d6ff259" } file = phd-tests/framework/src/test_vm/mod.rs line = 373 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a vm = migration_from_base_and_back_lifecycle_0 vm_id = 80db91b4-36f2-4be4-9022-5422e94c246a
892024-09-18T18:28:17.853ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 573 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a vm = migration_from_base_and_back_lifecycle_0 vm_id = 80db91b4-36f2-4be4-9022-5422e94c246a
902024-09-18T18:28:17.853ZINFOphd-runner: [MIGRATE_FROM - END] elapsed_milliseconds = 573 file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::migration_from_base_and_back source = migration_from_base_and_back source_id = 1ac4585f-0fce-463f-be7c-77197a03c024 target = phd_framework::test_vm target_id = 80db91b4-36f2-4be4-9022-5422e94c246a
912024-09-18T18:28:17.853ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9002 target = phd_framework::test_vm::server
922024-09-18T18:28:17.882ZINFOphd-runner: [VM CLEANUP - START] file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
932024-09-18T18:28:17.883ZDEBGphd-runner: [VM CLEANUP - EVENT] trying to ensure Propolis server VM is destroyed file = phd-tests/framework/src/test_vm/mod.rs line = 1040 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
942024-09-18T18:28:17.984ZDEBGphd-runner: [SERIAL CONSOLE TASK - EVENT] serial console command channel was closed file = phd-tests/framework/src/serial/mod.rs line = 213 target = phd_framework::serial
952024-09-18T18:28:17.984ZINFOphd-runner: [SERIAL CONSOLE TASK - END] elapsed_milliseconds = 5096 file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
962024-09-18T18:28:17.985ZINFOphd-runner: test phd_tests::migrate::from_base::migration_from_base_and_back ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "466c2a3b-14c6-4b8c-8502-45906d6ff259", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:17 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "466c2a3b-14c6-4b8c-8502-45906d6ff259" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1989:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:385:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:290:5 5: phd_framework::test_vm::TestVm::migrate_from::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:589:22 6: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 7: phd_framework::test_vm::TestVm::migrate_from::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:539:5 8: phd_framework::lifecycle::<impl phd_framework::Framework>::lifecycle_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/lifecycle.rs:95:26 9: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:83:10 10: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 11: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:52:5 12: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 13: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 14: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 15: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 16: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 17: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 18: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 19: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 20: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 21: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 22: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 23: __rust_try 24: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 25: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 26: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 27: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 28: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 29: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 30: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 31: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 32: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 33: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 34: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 35: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 36: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 37: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 38: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 39: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 40: std::thread::local::LocalKey<T>::try_with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12 41: std::thread::local::LocalKey<T>::with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9 42: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 43: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 44: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 45: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 46: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 47: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 48: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 49: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 50: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 51: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 52: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 53: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 54: __rust_try 55: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 56: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 57: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 58: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 59: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 60: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 61: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 62: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 63: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 64: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 65: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 66: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18 67: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17 68: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 69: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 70: __rust_try 71: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 72: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 73: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30 74: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 76: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 77: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17 78: _thrp_setup 79: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
972024-09-18T18:28:17.985ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
982024-09-18T18:28:18.346ZDEBGphd-runner: [VM CLEANUP - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
992024-09-18T18:28:18.484ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
1002024-09-18T18:28:18.484ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-6c097792-b380-45c4-99e6-f9ccff525ba7"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
1012024-09-18T18:28:18.519ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
1022024-09-18T18:28:18.519ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts@phd-454ca2f4-334e-48a4-bdc0-a0eefcaf1c52"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
1032024-09-18T18:28:18.545ZINFOphd-runner: [VM CLEANUP - END] elapsed_milliseconds = 662 file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 1ac4585f-0fce-463f-be7c-77197a03c024
1042024-09-18T18:28:18.545ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 72188 file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
1052024-09-18T18:28:18.545ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 560 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1062024-09-18T18:28:18.545ZINFOphd-runner: Starting test phd_tests::migrate::from_base::serial_history file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
1072024-09-18T18:28:18.545ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
1082024-09-18T18:28:18.545ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
1092024-09-18T18:28:18.545ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
1102024-09-18T18:28:18.545ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
1112024-09-18T18:28:18.545ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
1122024-09-18T18:28:18.545ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
1132024-09-18T18:28:18.545ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
1142024-09-18T18:28:18.564ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 709M 1.80T 709M /phd-artifacts\n target = phd_framework::zfs
1152024-09-18T18:28:18.564ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-dabc02b7-b0b0-4a2f-b382-295b24c87d30"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = snapshot
1162024-09-18T18:28:18.588ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-dabc02b7-b0b0-4a2f-b382-295b24c87d30", "phd-artifacts/phd-clone-fcd7c9c6-fb58-4f4a-8a7e-02e8693f4a04"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = clone
1172024-09-18T18:28:18.613ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-fcd7c9c6-fb58-4f4a-8a7e-02e8693f4a04"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
1182024-09-18T18:28:18.630ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-fcd7c9c6-fb58-4f4a-8a7e-02e8693f4a04 0B 1.80T 709M /phd-artifacts/phd-clone-fcd7c9c6-fb58-4f4a-8a7e-02e8693f4a04\n target = phd_framework::zfs
1192024-09-18T18:28:18.631ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1202024-09-18T18:28:18.631ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base
1212024-09-18T18:28:18.631ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server target = phd_framework::artifacts::store
1222024-09-18T18:28:18.631ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_serial_history_base.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server" target = phd_framework::test_vm::server
1232024-09-18T18:28:18.631ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/migration_serial_history_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_serial_history_base.stdout.log" target = phd_framework::server_log_mode
1242024-09-18T18:28:18.643ZINFOphd-runner: [NEW - EVENT] Launched server with pid 875 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
1252024-09-18T18:28:18.643ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 12 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1262024-09-18T18:28:18.643ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1272024-09-18T18:28:18.644ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1282024-09-18T18:28:19.248ZINFOphd-runner: [SERIAL CONSOLE TASK - START] file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
1292024-09-18T18:28:19.249ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] Started instance file = phd-tests/framework/src/test_vm/mod.rs instance_description.instance = Instance { disks: [], nics: [], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Pheidippides-managed VM", id: 84cbfc8b-b0b3-4675-863c-23d81fe463e5, image_id: 00000000-0000-0000-0000-000000000000, memory: 512, metadata: InstanceMetadata { project_id: 936fa5ca-a275-48d7-b6fb-3a2bd9d56d54, silo_id: a640620f-12f1-46b4-9720-d7a3d54a6917, sled_id: 24396d95-ab28-4cd1-a362-108e2b4677e6, sled_model: "pheidippes", sled_revision: 1, sled_serial: "24396d95-ab28-4cd1-a362-108e2b4677e6" }, name: "phd-vm-84cbfc8b-b0b3-4675-863c-23d81fe463e5", vcpus: 2 }, state: Creating } line = 428 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1302024-09-18T18:28:19.249ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 605 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1312024-09-18T18:28:19.249ZINFOphd-runner: [PUT_INSTANCE_STATE - START] file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1322024-09-18T18:28:19.249ZINFOphd-runner: [PUT_INSTANCE_STATE - EVENT] Requesting instance state change file = phd-tests/framework/src/test_vm/mod.rs line = 510 path = phd_tests::migrate::from_base::serial_history state = Run target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1332024-09-18T18:28:19.250ZINFOphd-runner: [PUT_INSTANCE_STATE - END] elapsed_milliseconds = 0 file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1342024-09-18T18:28:19.250ZINFOphd-runner: [WAIT_TO_BOOT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1352024-09-18T18:28:19.250ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] waiting for guest to boot file = phd-tests/framework/src/test_vm/mod.rs line = 711 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm timeout_duration = 300s vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1362024-09-18T18:28:19.250ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::serial_history step = WaitFor("localhost login: ") target = phd_framework::test_vm
1372024-09-18T18:28:19.250ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1382024-09-18T18:28:19.250ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1392024-09-18T18:28:23.434ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 4184 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1402024-09-18T18:28:23.434ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::serial_history step = WriteStr("root") target = phd_framework::test_vm
1412024-09-18T18:28:23.434ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::serial_history step = WaitFor("localhost:~#") target = phd_framework::test_vm
1422024-09-18T18:28:23.435ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1432024-09-18T18:28:23.435ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1442024-09-18T18:28:23.453ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 18 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1452024-09-18T18:28:23.454ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] Guest has booted file = phd-tests/framework/src/test_vm/mod.rs line = 759 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1462024-09-18T18:28:23.454ZINFOphd-runner: [WAIT_TO_BOOT - END] elapsed_milliseconds = 4203 file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1472024-09-18T18:28:23.454ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1482024-09-18T18:28:23.454ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1492024-09-18T18:28:23.464ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1502024-09-18T18:28:23.465ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1512024-09-18T18:28:23.465ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1522024-09-18T18:28:23.497ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 31 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1532024-09-18T18:28:23.501ZINFOphd-runner: [TEST - EVENT] migrating to new Propolis artifact for lifecycle test file = phd-tests/framework/src/lifecycle.rs line = 74 path = phd_tests::migrate::from_base::serial_history propolis_artifact = __DEFAULT_PROPOLIS target = phd_framework::lifecycle vm_name = migration_serial_history_base
1542024-09-18T18:28:23.501ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1552024-09-18T18:28:23.502ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__DEFAULT_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base_lifecycle_0
1562024-09-18T18:28:23.502ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /work/phd-test/propolis-server target = phd_framework::artifacts::store
1572024-09-18T18:28:23.502ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_serial_history_base_lifecycle_0.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9002 server_path = "/work/phd-test/propolis-server" target = phd_framework::test_vm::server
1582024-09-18T18:28:23.502ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/migration_serial_history_base_lifecycle_0.stderr.log" stdout_path = "/tmp/propolis-phd/migration_serial_history_base_lifecycle_0.stdout.log" target = phd_framework::server_log_mode
1592024-09-18T18:28:23.520ZINFOphd-runner: [NEW - EVENT] Launched server with pid 876 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
1602024-09-18T18:28:23.521ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 19 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
1612024-09-18T18:28:23.521ZINFOphd-runner: [MIGRATE_FROM - START] file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1622024-09-18T18:28:23.521ZINFOphd-runner: [MIGRATE_FROM - EVENT] Migrating from source at address 127.0.0.1:9000 file = phd-tests/framework/src/test_vm/mod.rs line = 572 migration_id = 65dae6b8-accf-4004-ae16-c04915dc7f5a path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e timeout_duration = 45s
1632024-09-18T18:28:23.521ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e vm = migration_serial_history_base_lifecycle_0 vm_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1642024-09-18T18:28:23.521ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9002/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e vm = migration_serial_history_base_lifecycle_0 vm_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1652024-09-18T18:28:23.954ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7452fb5c-6481-4733-8f20-50653908b74f", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:23 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7452fb5c-6481-4733-8f20-50653908b74f" } file = phd-tests/framework/src/test_vm/mod.rs line = 373 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e vm = migration_serial_history_base_lifecycle_0 vm_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1662024-09-18T18:28:23.954ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 433 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e vm = migration_serial_history_base_lifecycle_0 vm_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1672024-09-18T18:28:23.954ZINFOphd-runner: [MIGRATE_FROM - END] elapsed_milliseconds = 433 file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::serial_history source = migration_serial_history_base source_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5 target = phd_framework::test_vm target_id = 0f8954c2-9916-41c1-a173-16efdd7ed07e
1682024-09-18T18:28:23.954ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9002 target = phd_framework::test_vm::server
1692024-09-18T18:28:23.985ZINFOphd-runner: [VM CLEANUP - START] file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1702024-09-18T18:28:23.986ZDEBGphd-runner: [VM CLEANUP - EVENT] trying to ensure Propolis server VM is destroyed file = phd-tests/framework/src/test_vm/mod.rs line = 1040 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1712024-09-18T18:28:23.987ZDEBGphd-runner: [SERIAL CONSOLE TASK - EVENT] serial console command channel was closed file = phd-tests/framework/src/serial/mod.rs line = 213 target = phd_framework::serial
1722024-09-18T18:28:23.987ZINFOphd-runner: [SERIAL CONSOLE TASK - END] elapsed_milliseconds = 4738 file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
1732024-09-18T18:28:23.987ZINFOphd-runner: test phd_tests::migrate::from_base::serial_history ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "7452fb5c-6481-4733-8f20-50653908b74f", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:23 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "7452fb5c-6481-4733-8f20-50653908b74f" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1989:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:385:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:290:5 5: phd_framework::test_vm::TestVm::migrate_from::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:589:22 6: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 7: phd_framework::test_vm::TestVm::migrate_from::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:539:5 8: phd_framework::lifecycle::<impl phd_framework::Framework>::lifecycle_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/lifecycle.rs:95:26 9: phd_tests::migrate::run_serial_history_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:399:6 10: phd_tests::migrate::from_base::serial_history::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:46:10 11: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 12: phd_tests::migrate::from_base::serial_history::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:40:5 13: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 14: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 16: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 17: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 18: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 19: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 20: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 21: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 22: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 23: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 24: __rust_try 25: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 26: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 27: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 28: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 29: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 30: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 31: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 32: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 33: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 34: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 35: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 36: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 37: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 38: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 39: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 40: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 41: std::thread::local::LocalKey<T>::try_with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12 42: std::thread::local::LocalKey<T>::with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9 43: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 44: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 45: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 46: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 47: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 48: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 49: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 50: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 51: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 52: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 53: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 54: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 55: __rust_try 56: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 57: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 58: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 59: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 60: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 61: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 62: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 63: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 64: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 65: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 66: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 67: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18 68: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17 69: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 70: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 71: __rust_try 72: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 73: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 74: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30 75: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 76: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 77: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 78: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17 79: _thrp_setup 80: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
1742024-09-18T18:28:23.987ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1752024-09-18T18:28:24.297ZDEBGphd-runner: [VM CLEANUP - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1762024-09-18T18:28:24.423ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1772024-09-18T18:28:24.423ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-fcd7c9c6-fb58-4f4a-8a7e-02e8693f4a04"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1782024-09-18T18:28:24.461ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1792024-09-18T18:28:24.461ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts@phd-dabc02b7-b0b0-4a2f-b382-295b24c87d30"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1802024-09-18T18:28:24.491ZINFOphd-runner: [VM CLEANUP - END] elapsed_milliseconds = 505 file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 84cbfc8b-b0b3-4675-863c-23d81fe463e5
1812024-09-18T18:28:24.491ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 5945 file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
1822024-09-18T18:28:24.491ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 503 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1832024-09-18T18:28:24.491ZINFOphd-runner: Starting test phd_tests::migrate::from_base::can_migrate_from_base file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
1842024-09-18T18:28:24.491ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
1852024-09-18T18:28:24.491ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
1862024-09-18T18:28:24.491ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
1872024-09-18T18:28:24.491ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
1882024-09-18T18:28:24.491ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
1892024-09-18T18:28:24.491ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
1902024-09-18T18:28:24.491ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
1912024-09-18T18:28:24.509ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 710M 1.80T 709M /phd-artifacts\n target = phd_framework::zfs
1922024-09-18T18:28:24.509ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-b63c14a9-6843-459f-9190-4cca95f7a804"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = snapshot
1932024-09-18T18:28:24.532ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-b63c14a9-6843-459f-9190-4cca95f7a804", "phd-artifacts/phd-clone-c1dab74c-1a6b-44f2-b419-7ef9dc2a058b"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = clone
1942024-09-18T18:28:24.556ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-c1dab74c-1a6b-44f2-b419-7ef9dc2a058b"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
1952024-09-18T18:28:24.573ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-c1dab74c-1a6b-44f2-b419-7ef9dc2a058b 0B 1.80T 709M /phd-artifacts/phd-clone-c1dab74c-1a6b-44f2-b419-7ef9dc2a058b\n target = phd_framework::zfs
1962024-09-18T18:28:24.573ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
1972024-09-18T18:28:24.573ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base
1982024-09-18T18:28:24.573ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server target = phd_framework::artifacts::store
1992024-09-18T18:28:24.573ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/2a13bdc8e33c968b5133ba5d408d0849f32970a6b469b9b9f0be0185bc46f271/propolis-server" target = phd_framework::test_vm::server
2002024-09-18T18:28:24.573ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/migration_from_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base.stdout.log" target = phd_framework::server_log_mode
2012024-09-18T18:28:24.584ZINFOphd-runner: [NEW - EVENT] Launched server with pid 885 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
2022024-09-18T18:28:24.584ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2032024-09-18T18:28:24.584ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2042024-09-18T18:28:24.585ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2052024-09-18T18:28:25.231ZINFOphd-runner: [SERIAL CONSOLE TASK - START] file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
2062024-09-18T18:28:25.232ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] Started instance file = phd-tests/framework/src/test_vm/mod.rs instance_description.instance = Instance { disks: [], nics: [], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: "Pheidippides-managed VM", id: 66c036a3-84a3-41f5-9879-950dd12fd6c9, image_id: 00000000-0000-0000-0000-000000000000, memory: 512, metadata: InstanceMetadata { project_id: 58a26037-b2ed-4933-bd4b-34c7384b4aea, silo_id: 6e295296-b5c5-43f6-a60a-8d3cb626f4eb, sled_id: 37eb1662-0a26-42a8-90fb-8f59ff09cc89, sled_model: "pheidippes", sled_revision: 1, sled_serial: "37eb1662-0a26-42a8-90fb-8f59ff09cc89" }, name: "phd-vm-66c036a3-84a3-41f5-9879-950dd12fd6c9", vcpus: 2 }, state: Creating } line = 428 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2072024-09-18T18:28:25.232ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 647 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2082024-09-18T18:28:25.232ZINFOphd-runner: [PUT_INSTANCE_STATE - START] file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2092024-09-18T18:28:25.232ZINFOphd-runner: [PUT_INSTANCE_STATE - EVENT] Requesting instance state change file = phd-tests/framework/src/test_vm/mod.rs line = 510 path = phd_tests::migrate::from_base::can_migrate_from_base state = Run target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2102024-09-18T18:28:25.233ZINFOphd-runner: [PUT_INSTANCE_STATE - END] elapsed_milliseconds = 0 file = phd-tests/framework/src/test_vm/mod.rs line = 505 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2112024-09-18T18:28:25.233ZINFOphd-runner: [WAIT_TO_BOOT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2122024-09-18T18:28:25.233ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] waiting for guest to boot file = phd-tests/framework/src/test_vm/mod.rs line = 711 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = 300s vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2132024-09-18T18:28:25.233ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::can_migrate_from_base step = WaitFor("localhost login: ") target = phd_framework::test_vm
2142024-09-18T18:28:25.233ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2152024-09-18T18:28:25.233ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2162024-09-18T18:28:29.433ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 4199 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2172024-09-18T18:28:29.433ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::can_migrate_from_base step = WriteStr("root") target = phd_framework::test_vm
2182024-09-18T18:28:29.433ZDEBGphd-runner: [WAIT_TO_BOOT - EVENT] executing command in boot sequence file = phd-tests/framework/src/test_vm/mod.rs line = 719 path = phd_tests::migrate::from_base::can_migrate_from_base step = WaitFor("localhost:~#") target = phd_framework::test_vm
2192024-09-18T18:28:29.433ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2202024-09-18T18:28:29.433ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = CallerTimeout vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2212024-09-18T18:28:29.451ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 17 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2222024-09-18T18:28:29.451ZINFOphd-runner: [WAIT_TO_BOOT - EVENT] Guest has booted file = phd-tests/framework/src/test_vm/mod.rs line = 759 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2232024-09-18T18:28:29.451ZINFOphd-runner: [WAIT_TO_BOOT - END] elapsed_milliseconds = 4218 file = phd-tests/framework/src/test_vm/mod.rs line = 762 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2242024-09-18T18:28:29.451ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2252024-09-18T18:28:29.452ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2262024-09-18T18:28:29.462ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 10 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2272024-09-18T18:28:29.462ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2282024-09-18T18:28:29.462ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2292024-09-18T18:28:29.484ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 21 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2302024-09-18T18:28:29.484ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2312024-09-18T18:28:29.484ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2322024-09-18T18:28:29.495ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 11 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2332024-09-18T18:28:29.496ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2342024-09-18T18:28:29.496ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2352024-09-18T18:28:29.528ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 32 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2362024-09-18T18:28:29.529ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2372024-09-18T18:28:29.529ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(15s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2382024-09-18T18:28:29.540ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 11 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2392024-09-18T18:28:29.540ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - START] file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2402024-09-18T18:28:29.540ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - EVENT] Waiting for output on serial console file = phd-tests/framework/src/test_vm/mod.rs line = 784 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm timeout_duration = Explicit(300s) vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2412024-09-18T18:28:29.573ZINFOphd-runner: [WAIT_FOR_SERIAL_OUTPUT - END] elapsed_milliseconds = 32 file = phd-tests/framework/src/test_vm/mod.rs line = 777 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2422024-09-18T18:28:29.573ZINFOphd-runner: [TEST - EVENT] migrating to new Propolis artifact for lifecycle test file = phd-tests/framework/src/lifecycle.rs line = 74 path = phd_tests::migrate::from_base::can_migrate_from_base propolis_artifact = __DEFAULT_PROPOLIS target = phd_framework::lifecycle vm_name = migration_from_base
2432024-09-18T18:28:29.573ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2442024-09-18T18:28:29.573ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__DEFAULT_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 204 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base_lifecycle_0
2452024-09-18T18:28:29.574ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /work/phd-test/propolis-server target = phd_framework::artifacts::store
2462024-09-18T18:28:29.574ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base_lifecycle_0.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9002 server_path = "/work/phd-test/propolis-server" target = phd_framework::test_vm::server
2472024-09-18T18:28:29.574ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/migration_from_base_lifecycle_0.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_lifecycle_0.stdout.log" target = phd_framework::server_log_mode
2482024-09-18T18:28:29.592ZINFOphd-runner: [NEW - EVENT] Launched server with pid 886 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
2492024-09-18T18:28:29.593ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 19 file = phd-tests/framework/src/test_vm/mod.rs line = 186 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
2502024-09-18T18:28:29.593ZINFOphd-runner: [MIGRATE_FROM - START] file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2512024-09-18T18:28:29.593ZINFOphd-runner: [MIGRATE_FROM - EVENT] Migrating from source at address 127.0.0.1:9000 file = phd-tests/framework/src/test_vm/mod.rs line = 572 migration_id = 9b2af353-0f01-4e9e-9c3c-23daf43c5099 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad timeout_duration = 45s
2522024-09-18T18:28:29.593ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad vm = migration_from_base_lifecycle_0 vm_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2532024-09-18T18:28:29.594ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9002/instance/spec): error trying to connect: tcp connect error: Connection refused (os error 146) file = phd-tests/framework/src/test_vm/mod.rs line = 369 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad vm = migration_from_base_lifecycle_0 vm_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2542024-09-18T18:28:30.267ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "57a825b2-6381-40f0-91ad-fdc51c7ab453", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "57a825b2-6381-40f0-91ad-fdc51c7ab453" } file = phd-tests/framework/src/test_vm/mod.rs line = 373 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad vm = migration_from_base_lifecycle_0 vm_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2552024-09-18T18:28:30.267ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 674 file = phd-tests/framework/src/test_vm/mod.rs line = 290 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad vm = migration_from_base_lifecycle_0 vm_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2562024-09-18T18:28:30.268ZINFOphd-runner: [MIGRATE_FROM - END] elapsed_milliseconds = 674 file = phd-tests/framework/src/test_vm/mod.rs line = 539 path = phd_tests::migrate::from_base::can_migrate_from_base source = migration_from_base source_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9 target = phd_framework::test_vm target_id = 3130664e-bb28-4862-8f47-a78a5f770fad
2572024-09-18T18:28:30.268ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9002 target = phd_framework::test_vm::server
2582024-09-18T18:28:30.293ZINFOphd-runner: [VM CLEANUP - START] file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2592024-09-18T18:28:30.294ZDEBGphd-runner: [SERIAL CONSOLE TASK - EVENT] serial console command channel was closed file = phd-tests/framework/src/serial/mod.rs line = 213 target = phd_framework::serial
2602024-09-18T18:28:30.294ZINFOphd-runner: [SERIAL CONSOLE TASK - END] elapsed_milliseconds = 5063 file = phd-tests/framework/src/serial/mod.rs line = 200 target = phd_framework::serial
2612024-09-18T18:28:30.294ZINFOphd-runner: test phd_tests::migrate::from_base::can_migrate_from_base ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "57a825b2-6381-40f0-91ad-fdc51c7ab453", "content-length": "124", "date": "Wed, 18 Sep 2024 18:28:30 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "57a825b2-6381-40f0-91ad-fdc51c7ab453" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1989:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:385:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:290:5 5: phd_framework::test_vm::TestVm::migrate_from::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:589:22 6: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 7: phd_framework::test_vm::TestVm::migrate_from::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:539:5 8: phd_framework::lifecycle::<impl phd_framework::Framework>::lifecycle_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/lifecycle.rs:95:26 9: phd_tests::migrate::run_smoke_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:361:6 10: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:37:14 11: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 12: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:34:5 13: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 14: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 16: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 17: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 18: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 19: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 20: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 21: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 22: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 23: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 24: __rust_try 25: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 26: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 27: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 28: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 29: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 30: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 31: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 32: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 33: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 34: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 35: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 36: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 37: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 38: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 39: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 40: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 41: std::thread::local::LocalKey<T>::try_with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12 42: std::thread::local::LocalKey<T>::with at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9 43: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 44: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 45: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 46: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 47: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 48: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 49: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 50: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 51: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 52: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 53: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 54: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 55: __rust_try 56: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 57: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 58: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 59: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 60: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 61: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 62: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 63: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 64: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 65: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 66: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 67: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18 68: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17 69: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 70: std::panicking::try::do_call at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 71: __rust_try 72: std::panicking::try at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 73: std::panic::catch_unwind at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 74: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30 75: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 76: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 77: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 78: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17 79: _thrp_setup 80: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
2622024-09-18T18:28:30.294ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
2632024-09-18T18:28:30.294ZDEBGphd-runner: [VM CLEANUP - EVENT] trying to ensure Propolis server VM is destroyed file = phd-tests/framework/src/test_vm/mod.rs line = 1040 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2642024-09-18T18:28:30.800ZDEBGphd-runner: [VM CLEANUP - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2652024-09-18T18:28:30.933ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2662024-09-18T18:28:30.933ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-c1dab74c-1a6b-44f2-b419-7ef9dc2a058b"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2672024-09-18T18:28:30.964ZDEBGphd-runner: [VM CLEANUP - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2682024-09-18T18:28:30.964ZDEBGphd-runner: [VM CLEANUP - EVENT] executing ZFS command args = ["phd-artifacts@phd-b63c14a9-6843-459f-9190-4cca95f7a804"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2692024-09-18T18:28:30.989ZINFOphd-runner: [VM CLEANUP - END] elapsed_milliseconds = 696 file = phd-tests/framework/src/test_vm/mod.rs line = 1017 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 66c036a3-84a3-41f5-9879-950dd12fd6c9
2702024-09-18T18:28:30.989ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 6497 file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
2712024-09-18T18:28:30.989ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 695 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
2722024-09-18T18:28:30.989ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
2732024-09-18T18:28:30.989ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
274 
275 failures:
276 phd_tests::migrate::from_base::migration_from_base_and_back
277 phd_tests::migrate::from_base::serial_history
278 phd_tests::migrate::from_base::can_migrate_from_base
279 
280 test result: FAILED. 0 passed; 3 failed; 0 skipped; 0 not run; finished in 84.63s
281 
2822024-09-18T18:28:30.989ZDEBGphd-runner: phd_runner exit_code = 3 file = phd-tests/runner/src/main.rs line = 40 target = phd_runner