Max level shown:
12024-10-29T19:26:52.638ZINFOphd-runner: phd_runner file = phd-tests/runner/src/main.rs line = 35 runner_args = ProcessArgs { command: Run(RunOptions { propolis_server_cmd: "/work/phd-test/propolis-server", base_propolis_branch: Some("master"), base_propolis_commit: None, base_propolis_cmd: None, crucible_downstairs_cmd: None, crucible_downstairs_commit: Some(Auto), tmp_directory: "/tmp/propolis-phd", artifact_directory: Some("/phd-artifacts"), server_logging_mode: TmpFile, default_guest_cpus: 2, default_guest_memory_mib: 512, artifact_toml_path: "/work/phd-test/artifacts.toml", default_guest_artifact: "alpine", default_bootrom_artifact: "ovmf", include_filter: ["phd_tests::migrate::from_base"], exclude_filter: [], max_buildomat_wait_secs: 1200 }), disable_ansi: false, emit_bunyan: true } target = phd_runner
22024-10-29T19:26:52.640ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 2196, tv_nsec: 387363491 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
32024-10-29T19:26:52.640ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
42024-10-29T19:26:52.640ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = b7b9d5660b28ca5e865242b2bdecd032c0852d40 file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
52024-10-29T19:26:52.640ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/b7b9d5660b28ca5e865242b2bdecd032c0852d40/crucible-nightly.sha256.txt
62024-10-29T19:26:52.771ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
72024-10-29T19:26:53.183ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/e0c83fd0e0760eec1af306286c50081689d11a51/propolis-server.sha256.txt
82024-10-29T19:26:53.291ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
92024-10-29T19:26:53.292ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
102024-10-29T19:26:53.292ZINFOphd-runner: Running 3 test(s) file = phd-tests/runner/src/execute.rs line = 82 target = phd_runner::execute
112024-10-29T19:26:53.292ZINFOphd-runner: Starting test phd_tests::migrate::from_base::migration_from_base_and_back file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
122024-10-29T19:26:53.292ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
132024-10-29T19:26:53.292ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
142024-10-29T19:26:53.292ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
152024-10-29T19:26:53.293ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
162024-10-29T19:26:53.293ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
172024-10-29T19:26:53.911ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
182024-10-29T19:26:53.911ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 618 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store timeout = 600s
192024-10-29T19:27:21.759ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
202024-10-29T19:27:21.759ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
212024-10-29T19:27:21.785ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 2.01M 1.80T 1.91M /phd-artifacts\n target = phd_framework::zfs
222024-10-29T19:27:21.785ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d150390d-16c4-448c-9bbf-791bf13ac122"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = snapshot
232024-10-29T19:27:21.835ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d150390d-16c4-448c-9bbf-791bf13ac122", "phd-artifacts/phd-clone-a65c0d1c-7953-4861-81c1-46ebea7d47e1"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = clone
242024-10-29T19:27:21.855ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-a65c0d1c-7953-4861-81c1-46ebea7d47e1"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
252024-10-29T19:27:21.866ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-a65c0d1c-7953-4861-81c1-46ebea7d47e1 0B 1.80T 48.9M /phd-artifacts/phd-clone-a65c0d1c-7953-4861-81c1-46ebea7d47e1\n target = phd_framework::zfs
262024-10-29T19:27:21.867ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
272024-10-29T19:27:21.867ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back
282024-10-29T19:27:21.867ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server.tar.gz path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
292024-10-29T19:27:21.867ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/e0c83fd0e0760eec1af306286c50081689d11a51/propolis-server.tar.gz
302024-10-29T19:27:46.587ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store untar_path = propolis-server
312024-10-29T19:27:46.587ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 518 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
322024-10-29T19:27:59.701ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base_and_back.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server" target = phd_framework::test_vm::server
332024-10-29T19:27:59.701ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/migration_from_base_and_back.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_and_back.stdout.log" target = phd_framework::server_log_mode
342024-10-29T19:27:59.740ZINFOphd-runner: [NEW - EVENT] Launched server with pid 865 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
352024-10-29T19:27:59.740ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 37873 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
362024-10-29T19:27:59.741ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f01047eb-4db7-4c2c-b3c7-eea08d4ac374
372024-10-29T19:27:59.741ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 370 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f01047eb-4db7-4c2c-b3c7-eea08d4ac374
382024-10-29T19:28:00.440ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "94ee14c7-1f65-46be-8b7f-a0f29b0ea0d9", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:00 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "94ee14c7-1f65-46be-8b7f-a0f29b0ea0d9" } file = phd-tests/framework/src/test_vm/mod.rs line = 374 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f01047eb-4db7-4c2c-b3c7-eea08d4ac374
392024-10-29T19:28:00.441ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 700 file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = f01047eb-4db7-4c2c-b3c7-eea08d4ac374
402024-10-29T19:28:00.442ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
412024-10-29T19:28:00.466ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
422024-10-29T19:28:00.466ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-a65c0d1c-7953-4861-81c1-46ebea7d47e1"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
432024-10-29T19:28:00.586ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
442024-10-29T19:28:00.586ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-d150390d-16c4-448c-9bbf-791bf13ac122"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
452024-10-29T19:28:00.614ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 67321 file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
462024-10-29T19:28:00.712ZINFOphd-runner: test phd_tests::migrate::from_base::migration_from_base_and_back ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "94ee14c7-1f65-46be-8b7f-a0f29b0ea0d9", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:00 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "94ee14c7-1f65-46be-8b7f-a0f29b0ea0d9" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/result.rs:1987:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:386:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:292:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:462:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:446:32 7: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:56:25 8: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 9: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:52:5 10: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/future/future.rs:123:9 11: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 12: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 13: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 14: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 15: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 16: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 17: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 18: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 19: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 20: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 21: __rust_try 22: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 23: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 24: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 25: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 26: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 27: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 28: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 29: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 30: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 31: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 32: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 33: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 34: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 35: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 36: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 37: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 38: std::thread::local::LocalKey<T>::try_with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:283:12 39: std::thread::local::LocalKey<T>::with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:260:9 40: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 41: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 42: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 43: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 44: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 45: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 46: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 47: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 48: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 49: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 50: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 51: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 52: __rust_try 53: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 54: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 55: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 56: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 57: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 58: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 59: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 60: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 61: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 62: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 63: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 64: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/backtrace.rs:154:18 65: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:522:17 66: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 67: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 68: __rust_try 69: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 70: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 71: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:521:30 72: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:250:5 73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 75: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/pal/unix/thread.rs:105:17 76: _thrp_setup 77: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
472024-10-29T19:28:00.712ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
482024-10-29T19:28:00.712ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
492024-10-29T19:28:00.712ZINFOphd-runner: Starting test phd_tests::migrate::from_base::serial_history file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
502024-10-29T19:28:00.712ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
512024-10-29T19:28:00.712ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
522024-10-29T19:28:00.712ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
532024-10-29T19:28:00.712ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
542024-10-29T19:28:00.713ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
552024-10-29T19:28:00.713ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
562024-10-29T19:28:00.713ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
572024-10-29T19:28:00.737ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 696M 1.80T 696M /phd-artifacts\n target = phd_framework::zfs
582024-10-29T19:28:00.737ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-532cf1c4-8840-4f43-bf13-885190830364"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = snapshot
592024-10-29T19:28:00.761ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-532cf1c4-8840-4f43-bf13-885190830364", "phd-artifacts/phd-clone-80a9d06c-7b2b-445b-9c2d-102df79834dc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = clone
602024-10-29T19:28:00.787ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-80a9d06c-7b2b-445b-9c2d-102df79834dc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
612024-10-29T19:28:00.805ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-80a9d06c-7b2b-445b-9c2d-102df79834dc 0B 1.80T 696M /phd-artifacts/phd-clone-80a9d06c-7b2b-445b-9c2d-102df79834dc\n target = phd_framework::zfs
622024-10-29T19:28:00.805ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
632024-10-29T19:28:00.805ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base
642024-10-29T19:28:00.805ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server target = phd_framework::artifacts::store
652024-10-29T19:28:00.805ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_serial_history_base.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server" target = phd_framework::test_vm::server
662024-10-29T19:28:00.805ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/migration_serial_history_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_serial_history_base.stdout.log" target = phd_framework::server_log_mode
672024-10-29T19:28:00.816ZINFOphd-runner: [NEW - EVENT] Launched server with pid 873 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
682024-10-29T19:28:00.816ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 11 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
692024-10-29T19:28:00.817ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 8d84a8cf-7874-4ec7-9999-31b3e16a081f
702024-10-29T19:28:00.817ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 370 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 8d84a8cf-7874-4ec7-9999-31b3e16a081f
712024-10-29T19:28:01.232ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6389d0b8-273c-460d-a5d5-3716bb05ab61", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:01 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "6389d0b8-273c-460d-a5d5-3716bb05ab61" } file = phd-tests/framework/src/test_vm/mod.rs line = 374 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 8d84a8cf-7874-4ec7-9999-31b3e16a081f
722024-10-29T19:28:01.232ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 415 file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 8d84a8cf-7874-4ec7-9999-31b3e16a081f
732024-10-29T19:28:01.232ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
742024-10-29T19:28:01.255ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
752024-10-29T19:28:01.255ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-80a9d06c-7b2b-445b-9c2d-102df79834dc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
762024-10-29T19:28:01.293ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
772024-10-29T19:28:01.293ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-532cf1c4-8840-4f43-bf13-885190830364"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
782024-10-29T19:28:01.316ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 603 file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
792024-10-29T19:28:01.318ZINFOphd-runner: test phd_tests::migrate::from_base::serial_history ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "6389d0b8-273c-460d-a5d5-3716bb05ab61", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:01 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "6389d0b8-273c-460d-a5d5-3716bb05ab61" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/result.rs:1987:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:386:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:292:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:462:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:446:32 7: phd_tests::migrate::run_serial_history_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:377:21 8: phd_tests::migrate::from_base::serial_history::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:46:10 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::serial_history::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:40:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 22: __rust_try 23: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 24: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 53: __rust_try 54: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 55: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:522:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 69: __rust_try 70: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 71: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:521:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
802024-10-29T19:28:01.318ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
812024-10-29T19:28:01.318ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
822024-10-29T19:28:01.318ZINFOphd-runner: Starting test phd_tests::migrate::from_base::can_migrate_from_base file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
832024-10-29T19:28:01.318ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
842024-10-29T19:28:01.318ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
852024-10-29T19:28:01.318ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
862024-10-29T19:28:01.318ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
872024-10-29T19:28:01.318ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
882024-10-29T19:28:01.318ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
892024-10-29T19:28:01.318ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
902024-10-29T19:28:01.337ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 697M 1.80T 696M /phd-artifacts\n target = phd_framework::zfs
912024-10-29T19:28:01.337ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-b7d2c9a9-c40e-46a0-bb08-cc1e39cb65e2"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = snapshot
922024-10-29T19:28:01.361ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-b7d2c9a9-c40e-46a0-bb08-cc1e39cb65e2", "phd-artifacts/phd-clone-1962b6d2-aeaf-4896-af97-d306b06cfa4f"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = clone
932024-10-29T19:28:01.385ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-1962b6d2-aeaf-4896-af97-d306b06cfa4f"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
942024-10-29T19:28:01.402ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-1962b6d2-aeaf-4896-af97-d306b06cfa4f 0B 1.80T 696M /phd-artifacts/phd-clone-1962b6d2-aeaf-4896-af97-d306b06cfa4f\n target = phd_framework::zfs
952024-10-29T19:28:01.403ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
962024-10-29T19:28:01.403ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base
972024-10-29T19:28:01.403ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server target = phd_framework::artifacts::store
982024-10-29T19:28:01.403ZINFOphd-runner: [NEW - EVENT] Launching Propolis server config_toml_path = "/tmp/propolis-phd/migration_from_base.config.toml" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/d1369ebeeb1f65a1873782ed6e54296fa5bd31d5c7761a747c39dd55dea36d39/propolis-server" target = phd_framework::test_vm::server
992024-10-29T19:28:01.403ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/migration_from_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base.stdout.log" target = phd_framework::server_log_mode
1002024-10-29T19:28:01.414ZINFOphd-runner: [NEW - EVENT] Launched server with pid 881 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
1012024-10-29T19:28:01.414ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 11 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
1022024-10-29T19:28:01.414ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 328c218d-60f4-4184-b9d7-79538cbaa965
1032024-10-29T19:28:01.414ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 370 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 328c218d-60f4-4184-b9d7-79538cbaa965
1042024-10-29T19:28:02.123ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "3bf9d8e3-a4df-4491-a97a-886e71ceb3ba", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:02 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "3bf9d8e3-a4df-4491-a97a-886e71ceb3ba" } file = phd-tests/framework/src/test_vm/mod.rs line = 374 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 328c218d-60f4-4184-b9d7-79538cbaa965
1052024-10-29T19:28:02.124ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 709 file = phd-tests/framework/src/test_vm/mod.rs line = 292 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = 328c218d-60f4-4184-b9d7-79538cbaa965
1062024-10-29T19:28:02.124ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
1072024-10-29T19:28:02.144ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1082024-10-29T19:28:02.144ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-1962b6d2-aeaf-4896-af97-d306b06cfa4f"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1092024-10-29T19:28:02.180ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1102024-10-29T19:28:02.180ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-b7d2c9a9-c40e-46a0-bb08-cc1e39cb65e2"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1112024-10-29T19:28:02.203ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 885 file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
1122024-10-29T19:28:02.204ZINFOphd-runner: test phd_tests::migrate::from_base::can_migrate_from_base ... FAILED: Error Response: status: 400 Bad Request; headers: {"content-type": "application/json", "x-request-id": "3bf9d8e3-a4df-4491-a97a-886e71ceb3ba", "content-length": "159", "date": "Tue, 29 Oct 2024 19:28:02 GMT"}; value: Error { error_code: None, message: "unable to parse JSON body: properties: missing field `image_id` at line 1 column 973", request_id: "3bf9d8e3-a4df-4491-a97a-886e71ceb3ba" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/result.rs:1987:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:386:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:292:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:462:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:446:32 7: phd_tests::migrate::run_smoke_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:347:21 8: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:37:14 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:34:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 22: __rust_try 23: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 24: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:657:17 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 53: __rust_try 54: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 55: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:522:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:554:40 69: __rust_try 70: std::panicking::try at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panicking.rs:518:19 71: std::panic::catch_unwind at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/panic.rs:345:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/thread/mod.rs:521:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/alloc/src/boxed.rs:2231:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/f6e511eec7342f59a25f7c0534f1dbea00d01b14/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
1132024-10-29T19:28:02.205ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1142024-10-29T19:28:02.205ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1152024-10-29T19:28:02.205ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1162024-10-29T19:28:02.205ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
117 
118 failures:
119 phd_tests::migrate::from_base::migration_from_base_and_back
120 phd_tests::migrate::from_base::serial_history
121 phd_tests::migrate::from_base::can_migrate_from_base
122 
123 test result: FAILED. 0 passed; 3 failed; 0 skipped; 0 not run; finished in 68.91s
124 
1252024-10-29T19:28:02.205ZDEBGphd-runner: phd_runner exit_code = 3 file = phd-tests/runner/src/main.rs line = 40 target = phd_runner