Max level shown:
12024-12-17T18:57:34.667ZINFOphd-runner: phd_runner file = phd-tests/runner/src/main.rs line = 35 runner_args = ProcessArgs { command: Run(RunOptions { propolis_server_cmd: "/work/phd-test/propolis-server", base_propolis_branch: Some("master"), base_propolis_commit: None, base_propolis_cmd: None, crucible_downstairs_cmd: None, crucible_downstairs_commit: Some(Auto), tmp_directory: "/tmp/propolis-phd", artifact_directory: Some("/phd-artifacts"), server_logging_mode: TmpFile, default_guest_cpus: 2, default_guest_memory_mib: 512, artifact_toml_path: "/work/phd-test/artifacts.toml", default_guest_artifact: "alpine", default_bootrom_artifact: "ovmf", include_filter: ["phd_tests::migrate::from_base"], exclude_filter: [], max_buildomat_wait_secs: 1200 }), disable_ansi: false, emit_bunyan: true } target = phd_runner
22024-12-17T18:57:34.669ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 461, tv_nsec: 722182534 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
32024-12-17T18:57:34.669ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
42024-12-17T18:57:34.669ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = 5a41b826171c7d2a8412fa833377ab1df25ee8ec file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
52024-12-17T18:57:34.669ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/5a41b826171c7d2a8412fa833377ab1df25ee8ec/crucible-nightly.sha256.txt
62024-12-17T18:57:34.780ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
72024-12-17T18:57:35.222ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/bf824f23d84abdb7771b85ad681d23bd81ddaec4/propolis-server.sha256.txt
82024-12-17T18:57:35.326ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
92024-12-17T18:57:35.326ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
102024-12-17T18:57:35.326ZINFOphd-runner: Running 3 test(s) file = phd-tests/runner/src/execute.rs line = 82 target = phd_runner::execute
112024-12-17T18:57:35.327ZINFOphd-runner: Starting test phd_tests::migrate::from_base::migration_from_base_and_back file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
122024-12-17T18:57:35.327ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
132024-12-17T18:57:35.327ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
142024-12-17T18:57:35.327ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
152024-12-17T18:57:35.327ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
162024-12-17T18:57:35.327ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
172024-12-17T18:57:35.881ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
182024-12-17T18:57:35.881ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 618 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store timeout = 600s
192024-12-17T18:58:06.693ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
202024-12-17T18:58:06.693ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
212024-12-17T18:58:06.718ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 2.00M 1.80T 1.91M /phd-artifacts\n target = phd_framework::zfs
222024-12-17T18:58:06.719ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-aa7608d7-0e53-4738-b365-416586485fcf"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = snapshot
232024-12-17T18:58:06.766ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-aa7608d7-0e53-4738-b365-416586485fcf", "phd-artifacts/phd-clone-397d00d2-9a6a-49a6-831d-1f4687ed2105"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = clone
242024-12-17T18:58:06.783ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-397d00d2-9a6a-49a6-831d-1f4687ed2105"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
252024-12-17T18:58:06.794ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-397d00d2-9a6a-49a6-831d-1f4687ed2105 0B 1.80T 48.9M /phd-artifacts/phd-clone-397d00d2-9a6a-49a6-831d-1f4687ed2105\n target = phd_framework::zfs
262024-12-17T18:58:06.795ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
272024-12-17T18:58:06.795ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back
282024-12-17T18:58:06.795ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server.tar.gz path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
292024-12-17T18:58:06.795ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/bf824f23d84abdb7771b85ad681d23bd81ddaec4/propolis-server.tar.gz
302024-12-17T18:58:30.761ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store untar_path = propolis-server
312024-12-17T18:58:30.761ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 518 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
322024-12-17T18:58:43.508ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server" target = phd_framework::test_vm::server
332024-12-17T18:58:43.509ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/migration_from_base_and_back.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base_and_back.stdout.log" target = phd_framework::server_log_mode
342024-12-17T18:58:43.550ZINFOphd-runner: [NEW - EVENT] Launched server with pid 867 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
352024-12-17T18:58:43.550ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 36755 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
362024-12-17T18:58:43.550ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 0b6e04e0-6f7f-4e88-82e6-422d7182477f
372024-12-17T18:58:43.551ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 0b6e04e0-6f7f-4e88-82e6-422d7182477f
382024-12-17T18:58:43.853ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 0b6e04e0-6f7f-4e88-82e6-422d7182477f
392024-12-17T18:58:44.850ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 0b6e04e0-6f7f-4e88-82e6-422d7182477f
402024-12-17T18:58:44.851ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 1301 file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = 0b6e04e0-6f7f-4e88-82e6-422d7182477f
412024-12-17T18:58:44.851ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
422024-12-17T18:58:44.875ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
432024-12-17T18:58:44.876ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-397d00d2-9a6a-49a6-831d-1f4687ed2105"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
442024-12-17T18:58:44.974ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
452024-12-17T18:58:44.974ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-aa7608d7-0e53-4738-b365-416586485fcf"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
462024-12-17T18:58:45.007ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 69680 file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
472024-12-17T18:58:45.107ZINFOphd-runner: test phd_tests::migrate::from_base::migration_from_base_and_back ... FAILED: Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) Caused by: 0: error sending request for url (http://127.0.0.1:9000/instance/spec) 1: client error (Connect) 2: tcp connect error: Connection refused (os error 146) 3: Connection refused (os error 146) Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:365:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:271:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:441:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:425:32 7: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:56:25 8: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 9: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:52:5 10: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 11: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 12: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 13: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 14: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 15: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 16: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 17: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 18: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 19: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 20: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 21: __rust_try 22: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 23: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 24: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 25: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 26: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 27: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 28: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 29: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 30: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:594:13 31: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 32: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 33: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 34: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 35: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 36: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 37: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 38: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 39: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 40: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 41: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 42: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 43: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 44: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 45: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 46: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 47: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 48: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 49: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 50: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 51: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 52: __rust_try 53: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 54: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 55: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 56: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 57: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 58: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 59: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 60: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 61: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 62: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 63: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 64: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 65: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 66: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 67: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 68: __rust_try 69: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 70: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 71: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 72: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 76: _thrp_setup 77: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
482024-12-17T18:58:45.107ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
492024-12-17T18:58:45.107ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
502024-12-17T18:58:45.107ZINFOphd-runner: Starting test phd_tests::migrate::from_base::serial_history file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
512024-12-17T18:58:45.107ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
522024-12-17T18:58:45.107ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
532024-12-17T18:58:45.107ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
542024-12-17T18:58:45.107ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
552024-12-17T18:58:45.107ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
562024-12-17T18:58:45.108ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
572024-12-17T18:58:45.108ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
582024-12-17T18:58:45.135ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 696M 1.80T 694M /phd-artifacts\n target = phd_framework::zfs
592024-12-17T18:58:45.135ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-80252e37-d980-41ea-aed1-435b90529e73"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = snapshot
602024-12-17T18:58:45.157ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-80252e37-d980-41ea-aed1-435b90529e73", "phd-artifacts/phd-clone-cf50b50f-fdc0-45ce-bdfa-b2fd2ca35a05"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = clone
612024-12-17T18:58:45.183ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-cf50b50f-fdc0-45ce-bdfa-b2fd2ca35a05"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
622024-12-17T18:58:45.200ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-cf50b50f-fdc0-45ce-bdfa-b2fd2ca35a05 0B 1.80T 694M /phd-artifacts/phd-clone-cf50b50f-fdc0-45ce-bdfa-b2fd2ca35a05\n target = phd_framework::zfs
632024-12-17T18:58:45.201ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
642024-12-17T18:58:45.201ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base
652024-12-17T18:58:45.201ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server target = phd_framework::artifacts::store
662024-12-17T18:58:45.201ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server" target = phd_framework::test_vm::server
672024-12-17T18:58:45.201ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/migration_serial_history_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_serial_history_base.stdout.log" target = phd_framework::server_log_mode
682024-12-17T18:58:45.213ZINFOphd-runner: [NEW - EVENT] Launched server with pid 875 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
692024-12-17T18:58:45.213ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 12 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
702024-12-17T18:58:45.214ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = a12dc5e7-0c9b-4b20-85de-76e1494defc8
712024-12-17T18:58:45.214ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = a12dc5e7-0c9b-4b20-85de-76e1494defc8
722024-12-17T18:58:45.894ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = a12dc5e7-0c9b-4b20-85de-76e1494defc8
732024-12-17T18:58:46.452ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = a12dc5e7-0c9b-4b20-85de-76e1494defc8
742024-12-17T18:58:46.453ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 1239 file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = a12dc5e7-0c9b-4b20-85de-76e1494defc8
752024-12-17T18:58:46.453ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
762024-12-17T18:58:46.479ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
772024-12-17T18:58:46.479ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-cf50b50f-fdc0-45ce-bdfa-b2fd2ca35a05"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
782024-12-17T18:58:46.519ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
792024-12-17T18:58:46.519ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-80252e37-d980-41ea-aed1-435b90529e73"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
802024-12-17T18:58:46.545ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 1438 file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
812024-12-17T18:58:46.547ZINFOphd-runner: test phd_tests::migrate::from_base::serial_history ... FAILED: Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) Caused by: 0: error sending request for url (http://127.0.0.1:9000/instance/spec) 1: client error (Connect) 2: tcp connect error: Connection refused (os error 146) 3: Connection refused (os error 146) Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:365:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:271:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:441:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:425:32 7: phd_tests::migrate::run_serial_history_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:327:21 8: phd_tests::migrate::from_base::serial_history::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:46:10 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::serial_history::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:40:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 22: __rust_try 23: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 24: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:594:13 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 53: __rust_try 54: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 55: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 69: __rust_try 70: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 71: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
822024-12-17T18:58:46.547ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
832024-12-17T18:58:46.547ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
842024-12-17T18:58:46.547ZINFOphd-runner: Starting test phd_tests::migrate::from_base::can_migrate_from_base file = phd-tests/runner/src/execute.rs line = 90 target = phd_runner::execute
852024-12-17T18:58:46.547ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
862024-12-17T18:58:46.547ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
872024-12-17T18:58:46.547ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
882024-12-17T18:58:46.548ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
892024-12-17T18:58:46.548ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
902024-12-17T18:58:46.548ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
912024-12-17T18:58:46.548ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
922024-12-17T18:58:46.565ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 696M 1.80T 694M /phd-artifacts\n target = phd_framework::zfs
932024-12-17T18:58:46.566ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-8f8c8676-5b0c-4e59-8bbd-b21f74514e1c"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = snapshot
942024-12-17T18:58:46.588ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-8f8c8676-5b0c-4e59-8bbd-b21f74514e1c", "phd-artifacts/phd-clone-d37d3bdc-6c4c-4a7b-ae1f-523e2867aee3"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = clone
952024-12-17T18:58:46.611ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-d37d3bdc-6c4c-4a7b-ae1f-523e2867aee3"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
962024-12-17T18:58:46.627ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-d37d3bdc-6c4c-4a7b-ae1f-523e2867aee3 0B 1.80T 694M /phd-artifacts/phd-clone-d37d3bdc-6c4c-4a7b-ae1f-523e2867aee3\n target = phd_framework::zfs
972024-12-17T18:58:46.628ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
982024-12-17T18:58:46.628ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 206 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base
992024-12-17T18:58:46.628ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server target = phd_framework::artifacts::store
1002024-12-17T18:58:46.628ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/f757ad39f1557884b610bdda4e6bfe96b50b750b4b76be3b1065d8b433097df4/propolis-server" target = phd_framework::test_vm::server
1012024-12-17T18:58:46.628ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/migration_from_base.stderr.log" stdout_path = "/tmp/propolis-phd/migration_from_base.stdout.log" target = phd_framework::server_log_mode
1022024-12-17T18:58:46.639ZINFOphd-runner: [NEW - EVENT] Launched server with pid 883 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
1032024-12-17T18:58:46.639ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 11 file = phd-tests/framework/src/test_vm/mod.rs line = 188 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
1042024-12-17T18:58:46.639ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = eb4ce440-7707-4316-b57d-1deee5538ecc
1052024-12-17T18:58:46.640ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = eb4ce440-7707-4316-b57d-1deee5538ecc
1062024-12-17T18:58:46.913ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = eb4ce440-7707-4316-b57d-1deee5538ecc
1072024-12-17T18:58:47.771ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) file = phd-tests/framework/src/test_vm/mod.rs line = 349 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = eb4ce440-7707-4316-b57d-1deee5538ecc
1082024-12-17T18:58:47.772ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 1132 file = phd-tests/framework/src/test_vm/mod.rs line = 271 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = eb4ce440-7707-4316-b57d-1deee5538ecc
1092024-12-17T18:58:47.772ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
1102024-12-17T18:58:47.793ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1112024-12-17T18:58:47.793ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-d37d3bdc-6c4c-4a7b-ae1f-523e2867aee3"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1122024-12-17T18:58:47.831ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1132024-12-17T18:58:47.831ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-8f8c8676-5b0c-4e59-8bbd-b21f74514e1c"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1142024-12-17T18:58:47.858ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 1310 file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
1152024-12-17T18:58:47.859ZINFOphd-runner: test phd_tests::migrate::from_base::can_migrate_from_base ... FAILED: Communication Error: error sending request for url (http://127.0.0.1:9000/instance/spec) Caused by: 0: error sending request for url (http://127.0.0.1:9000/instance/spec) 1: client error (Connect) 2: tcp connect error: Connection refused (os error 146) 3: Connection refused (os error 146) Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/result.rs:2010:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:365:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:271:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:441:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:425:32 7: phd_tests::migrate::run_smoke_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:297:21 8: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:37:14 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:34:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/future/future.rs:123:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/macros/select.rs:557:49 14: <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/future/poll_fn.rs:58:9 15: phd_runner::execute::run_tests_with_ctx::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:105:13 16: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 17: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 18: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 19: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 20: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 21: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 22: __rust_try 23: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 24: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 25: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 26: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 27: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 28: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 29: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 30: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:427:9 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:594:13 32: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:107:5 33: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/coop.rs:73:5 34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:593:9 35: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:544:24 36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:509:21 37: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/scoped.rs:40:9 38: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:26 39: std::thread::local::LocalKey<T>::try_with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:283:12 40: std::thread::local::LocalKey<T>::with at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/local.rs:260:9 41: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context.rs:180:9 42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:504:9 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/context/runtime.rs:65:16 44: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:496:5 45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/scheduler/multi_thread/worker.rs:462:45 46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/task.rs:42:21 47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:328:17 48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/loom/std/unsafe_cell.rs:16:9 49: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/core.rs:317:13 50: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:485:19 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 52: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 53: __rust_try 54: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 55: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 56: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:473:18 57: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:208:27 58: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/harness.rs:153:15 59: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:271:5 60: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/raw.rs:201:18 61: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/task/mod.rs:464:9 62: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:160:9 63: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:518:17 64: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.3/src/runtime/blocking/pool.rs:476:13 65: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/backtrace.rs:154:18 66: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:538:17 67: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/panic/unwind_safe.rs:272:9 68: std::panicking::try::do_call at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:557:40 69: __rust_try 70: std::panicking::try at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panicking.rs:520:19 71: std::panic::catch_unwind at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/panic.rs:358:14 72: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/thread/mod.rs:537:30 73: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/core/src/ops/function.rs:250:5 74: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 75: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/alloc/src/boxed.rs:2454:9 76: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf/library/std/src/sys/pal/unix/thread.rs:105:17 77: _thrp_setup 78: <unknown> file = phd-tests/runner/src/execute.rs line = 129 target = phd_runner::execute
1162024-12-17T18:58:47.859ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1172024-12-17T18:58:47.859ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1182024-12-17T18:58:47.859ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1192024-12-17T18:58:47.859ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
120 
121 failures:
122 phd_tests::migrate::from_base::migration_from_base_and_back
123 phd_tests::migrate::from_base::serial_history
124 phd_tests::migrate::from_base::can_migrate_from_base
125 
126 test result: FAILED. 0 passed; 3 failed; 0 skipped; 0 not run; finished in 72.53s
127 
1282024-12-17T18:58:47.859ZDEBGphd-runner: phd_runner exit_code = 3 file = phd-tests/runner/src/main.rs line = 40 target = phd_runner