Max level shown:
12024-12-18T00:36:12.002ZINFOphd-runner: phd_runner file = phd-tests/runner/src/main.rs line = 35 runner_args = ProcessArgs { command: Run(RunOptions { propolis_server_cmd: "/work/phd-test/propolis-server", base_propolis_branch: Some("master"), base_propolis_commit: None, base_propolis_cmd: None, crucible_downstairs_cmd: None, crucible_downstairs_commit: Some(Auto), tmp_directory: "/tmp/propolis-phd", artifact_directory: Some("/phd-artifacts"), server_logging_mode: TmpFile, default_guest_cpus: 2, default_guest_memory_mib: 512, artifact_toml_path: "/work/phd-test/artifacts.toml", default_guest_artifact: "alpine", default_bootrom_artifact: "ovmf", include_filter: ["phd_tests::migrate::from_base"], exclude_filter: [], max_buildomat_wait_secs: 1200 }), disable_ansi: false, emit_bunyan: true } target = phd_runner
22024-12-18T00:36:12.004ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 330, tv_nsec: 322391657 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
32024-12-18T00:36:12.004ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
42024-12-18T00:36:12.004ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = 5a41b826171c7d2a8412fa833377ab1df25ee8ec file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
52024-12-18T00:36:12.004ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/5a41b826171c7d2a8412fa833377ab1df25ee8ec/crucible-nightly.sha256.txt
62024-12-18T00:36:12.132ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
72024-12-18T00:36:12.577ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/263fc2807d66bb0182057a898d2c2041a3784a3f/propolis-server.sha256.txt
82024-12-18T00:36:12.680ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
92024-12-18T00:36:12.680ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
102024-12-18T00:36:12.681ZINFOphd-runner: Running 3 test(s) file = phd-tests/runner/src/execute.rs line = 82 target = phd_runner::execute
112024-12-18T00:36:12.681ZINFOphd-runner: Starting test phd_tests::migrate::from_base::migration_from_base_and_back file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
122024-12-18T00:36:12.681ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
132024-12-18T00:36:12.681ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
142024-12-18T00:36:12.681ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
152024-12-18T00:36:12.681ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
162024-12-18T00:36:12.682ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
172024-12-18T00:36:13.248ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
182024-12-18T00:36:13.248ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 618 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store timeout = 600s
192024-12-18T00:36:43.714ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
202024-12-18T00:36:43.714ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
212024-12-18T00:36:43.739ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 2M 1.80T 1.91M /phd-artifacts\n target = phd_framework::zfs
222024-12-18T00:36:43.739ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-614d9b79-c84a-4e86-8672-33b337eebb47"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = snapshot
232024-12-18T00:36:43.768ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-614d9b79-c84a-4e86-8672-33b337eebb47", "phd-artifacts/phd-clone-edf69733-4919-47bc-986d-f155ebcde37d"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = clone
242024-12-18T00:36:43.787ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-edf69733-4919-47bc-986d-f155ebcde37d"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
252024-12-18T00:36:43.799ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-edf69733-4919-47bc-986d-f155ebcde37d 0B 1.80T 48.9M /phd-artifacts/phd-clone-edf69733-4919-47bc-986d-f155ebcde37d\n target = phd_framework::zfs
262024-12-18T00:36:43.800ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
272024-12-18T00:36:43.800ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 209 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back
282024-12-18T00:36:43.800ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server.tar.gz path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
292024-12-18T00:36:43.800ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/263fc2807d66bb0182057a898d2c2041a3784a3f/propolis-server.tar.gz
302024-12-18T00:37:07.685ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store untar_path = propolis-server
312024-12-18T00:37:07.685ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 518 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
322024-12-18T00:37:20.254ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server" target = phd_framework::test_vm::server
332024-12-18T00:37:20.254ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/migration_from_base_and_back.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_and_back.stdout.log" target = phd_framework::server_log_mode
342024-12-18T00:37:20.302ZINFOphd-runner: [NEW - EVENT] Launched server with pid 867 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
352024-12-18T00:37:20.303ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 36502 file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
362024-12-18T00:37:20.303ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f1baa94b-e9d8-4b36-bb3b-5ca325addfaf
372024-12-18T00:37:20.304ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 318 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f1baa94b-e9d8-4b36-bb3b-5ca325addfaf
382024-12-18T00:37:20.871ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "19ce20d2-4948-44f2-8cd6-fa684cce4691", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:20 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "19ce20d2-4948-44f2-8cd6-fa684cce4691" } file = phd-tests/framework/src/test_vm/mod.rs line = 322 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f1baa94b-e9d8-4b36-bb3b-5ca325addfaf
392024-12-18T00:37:20.872ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 569 file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f1baa94b-e9d8-4b36-bb3b-5ca325addfaf
402024-12-18T00:37:20.872ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
412024-12-18T00:37:20.899ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
422024-12-18T00:37:20.899ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-edf69733-4919-47bc-986d-f155ebcde37d"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
432024-12-18T00:37:20.966ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
442024-12-18T00:37:20.966ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-614d9b79-c84a-4e86-8672-33b337eebb47"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
452024-12-18T00:37:20.993ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 68311 file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
462024-12-18T00:37:21.089ZINFOphd-runner: test phd_tests::migrate::from_base::migration_from_base_and_back ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "19ce20d2-4948-44f2-8cd6-fa684cce4691", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:20 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "19ce20d2-4948-44f2-8cd6-fa684cce4691" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:334:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:274:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:406:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:394:32 7: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:56:25 8: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 9: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:52:5 10: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 11: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 12: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 13: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 14: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 15: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 16: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 17: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 18: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 19: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 20: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 21: __rust_try 22: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 23: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 24: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 25: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 26: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 27: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 28: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 29: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 30: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 31: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 32: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 33: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 34: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 35: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 36: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 37: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 38: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 39: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 40: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 41: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 42: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 43: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 44: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 45: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 46: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 47: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 48: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 49: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 50: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 51: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 52: __rust_try 53: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 54: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 55: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 56: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 57: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 58: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 59: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 60: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 61: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 62: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 63: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 64: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 65: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 66: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 67: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 68: __rust_try 69: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 70: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 71: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 72: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 76: _thrp_setup 77: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
472024-12-18T00:37:21.089ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
482024-12-18T00:37:21.089ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
492024-12-18T00:37:21.089ZINFOphd-runner: Starting test phd_tests::migrate::from_base::serial_history file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
502024-12-18T00:37:21.089ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
512024-12-18T00:37:21.089ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
522024-12-18T00:37:21.090ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
532024-12-18T00:37:21.090ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
542024-12-18T00:37:21.090ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
552024-12-18T00:37:21.090ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
562024-12-18T00:37:21.090ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
572024-12-18T00:37:21.116ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 679M 1.80T 679M /phd-artifacts\n target = phd_framework::zfs
582024-12-18T00:37:21.116ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d529264e-6b83-4ab3-b95a-383ce92284b3"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = snapshot
592024-12-18T00:37:21.141ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d529264e-6b83-4ab3-b95a-383ce92284b3", "phd-artifacts/phd-clone-3e8a6284-6a5c-47d4-98f9-565a20f33fad"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = clone
602024-12-18T00:37:21.164ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-3e8a6284-6a5c-47d4-98f9-565a20f33fad"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
612024-12-18T00:37:21.180ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-3e8a6284-6a5c-47d4-98f9-565a20f33fad 0B 1.80T 679M /phd-artifacts/phd-clone-3e8a6284-6a5c-47d4-98f9-565a20f33fad\n target = phd_framework::zfs
622024-12-18T00:37:21.181ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
632024-12-18T00:37:21.181ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 209 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base
642024-12-18T00:37:21.181ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server target = phd_framework::artifacts::store
652024-12-18T00:37:21.181ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server" target = phd_framework::test_vm::server
662024-12-18T00:37:21.181ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/migration_serial_history_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_serial_history_base.stdout.log" target = phd_framework::server_log_mode
672024-12-18T00:37:21.192ZINFOphd-runner: [NEW - EVENT] Launched server with pid 875 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
682024-12-18T00:37:21.193ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 12 file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
692024-12-18T00:37:21.193ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 72bf6f9b-f063-4643-8c28-0a848b158481
702024-12-18T00:37:21.193ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 318 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 72bf6f9b-f063-4643-8c28-0a848b158481
712024-12-18T00:37:21.634ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6877d98f-d283-4822-b177-19db4b5ab41e", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:21 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "6877d98f-d283-4822-b177-19db4b5ab41e" } file = phd-tests/framework/src/test_vm/mod.rs line = 322 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 72bf6f9b-f063-4643-8c28-0a848b158481
722024-12-18T00:37:21.634ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 441 file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 72bf6f9b-f063-4643-8c28-0a848b158481
732024-12-18T00:37:21.634ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
742024-12-18T00:37:21.661ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
752024-12-18T00:37:21.661ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-3e8a6284-6a5c-47d4-98f9-565a20f33fad"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
762024-12-18T00:37:21.699ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
772024-12-18T00:37:21.699ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d529264e-6b83-4ab3-b95a-383ce92284b3"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
782024-12-18T00:37:21.727ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 637 file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
792024-12-18T00:37:21.728ZINFOphd-runner: test phd_tests::migrate::from_base::serial_history ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6877d98f-d283-4822-b177-19db4b5ab41e", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:21 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "6877d98f-d283-4822-b177-19db4b5ab41e" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:334:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:274:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:406:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:394:32 7: phd_tests::migrate::run_serial_history_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:327:21 8: phd_tests::migrate::from_base::serial_history::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:46:10 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::serial_history::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:40:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 22: __rust_try 23: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 24: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 53: __rust_try 54: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 55: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 69: __rust_try 70: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 71: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
802024-12-18T00:37:21.728ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
812024-12-18T00:37:21.728ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
822024-12-18T00:37:21.728ZINFOphd-runner: Starting test phd_tests::migrate::from_base::can_migrate_from_base file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
832024-12-18T00:37:21.728ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
842024-12-18T00:37:21.728ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
852024-12-18T00:37:21.729ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
862024-12-18T00:37:21.729ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
872024-12-18T00:37:21.729ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
882024-12-18T00:37:21.729ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
892024-12-18T00:37:21.729ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
902024-12-18T00:37:21.748ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 681M 1.80T 679M /phd-artifacts\n target = phd_framework::zfs
912024-12-18T00:37:21.748ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-12b3aa51-b3fb-493d-ba6f-32c2064cc8db"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = snapshot
922024-12-18T00:37:21.770ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-12b3aa51-b3fb-493d-ba6f-32c2064cc8db", "phd-artifacts/phd-clone-8b41e842-8527-4b2c-9e6f-1acbcb7d0560"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = clone
932024-12-18T00:37:21.792ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-8b41e842-8527-4b2c-9e6f-1acbcb7d0560"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
942024-12-18T00:37:21.810ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-8b41e842-8527-4b2c-9e6f-1acbcb7d0560 0B 1.80T 679M /phd-artifacts/phd-clone-8b41e842-8527-4b2c-9e6f-1acbcb7d0560\n target = phd_framework::zfs
952024-12-18T00:37:21.811ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
962024-12-18T00:37:21.811ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 209 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base
972024-12-18T00:37:21.811ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server target = phd_framework::artifacts::store
982024-12-18T00:37:21.811ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/4735a63415554f00ce3617078ab6ca3f4dc47afb9090e7f609cb6db923b405c8/propolis-server" target = phd_framework::test_vm::server
992024-12-18T00:37:21.811ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/migration_from_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base.stdout.log" target = phd_framework::server_log_mode
1002024-12-18T00:37:21.823ZINFOphd-runner: [NEW - EVENT] Launched server with pid 883 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
1012024-12-18T00:37:21.823ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 12 file = phd-tests/framework/src/test_vm/mod.rs line = 191 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
1022024-12-18T00:37:21.824ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = bd207308-45db-4eb4-b1bc-b2f66916dccb
1032024-12-18T00:37:21.824ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 318 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = bd207308-45db-4eb4-b1bc-b2f66916dccb
1042024-12-18T00:37:22.325ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6236578f-f05b-452c-992b-318c4ba92fe6", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:22 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "6236578f-f05b-452c-992b-318c4ba92fe6" } file = phd-tests/framework/src/test_vm/mod.rs line = 322 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = bd207308-45db-4eb4-b1bc-b2f66916dccb
1052024-12-18T00:37:22.325ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 501 file = phd-tests/framework/src/test_vm/mod.rs line = 274 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = bd207308-45db-4eb4-b1bc-b2f66916dccb
1062024-12-18T00:37:22.325ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
1072024-12-18T00:37:22.347ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1082024-12-18T00:37:22.347ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-8b41e842-8527-4b2c-9e6f-1acbcb7d0560"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1092024-12-18T00:37:22.384ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1102024-12-18T00:37:22.384ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-12b3aa51-b3fb-493d-ba6f-32c2064cc8db"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1112024-12-18T00:37:22.409ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 680 file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
1122024-12-18T00:37:22.410ZINFOphd-runner: test phd_tests::migrate::from_base::can_migrate_from_base ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6236578f-f05b-452c-992b-318c4ba92fe6", "content-length": "144", "date": "Wed, 18 Dec 2024 00:37:22 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: missing field `vcpus` at line 1 column 976", request_id: "6236578f-f05b-452c-992b-318c4ba92fe6" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:334:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:274:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:406:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:394:32 7: phd_tests::migrate::run_smoke_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:297:21 8: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:37:14 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:34:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 22: __rust_try 23: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 24: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 53: __rust_try 54: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 55: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 69: __rust_try 70: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 71: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
1132024-12-18T00:37:22.410ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1142024-12-18T00:37:22.410ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1152024-12-18T00:37:22.410ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1162024-12-18T00:37:22.410ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
117 
118 failures:
119 phd_tests::migrate::from_base::migration_from_base_and_back
120 phd_tests::migrate::from_base::serial_history
121 phd_tests::migrate::from_base::can_migrate_from_base
122 
123 test result: FAILED. 0 passed; 3 failed; 0 skipped; 0 not run; finished in 69.73s
124 
1252024-12-18T00:37:22.410ZDEBGphd-runner: phd_runner exit_code = 3 file = phd-tests/runner/src/main.rs line = 40 target = phd_runner