Max level shown:
12025-03-10T21:07:36.152ZINFOphd-runner: phd_runner file = phd-tests/runner/src/main.rs line = 36 runner_args = ProcessArgs { command: Run(RunOptions { propolis_server_cmd: "/work/phd-test/propolis-server", base_propolis_branch: Some("master"), base_propolis_commit: None, base_propolis_cmd: None, crucible_downstairs_cmd: None, crucible_downstairs_commit: Some(Auto), tmp_directory: "/tmp/propolis-phd", artifact_directory: Some("/phd-artifacts"), server_logging_mode: TmpFile, parallelism: Some(2), default_guest_cpus: 2, default_guest_memory_mib: 512, artifact_toml_path: "/work/phd-test/artifacts.toml", default_guest_artifact: "alpine", default_bootrom_artifact: "ovmf", include_filter: ["phd_tests::migrate::from_base"], exclude_filter: [], max_buildomat_wait_secs: 1200 }), disable_ansi: false, emit_bunyan: true } target = phd_runner
22025-03-10T21:07:36.153ZINFOphd-runner: running tests with max parallelism of 2 file = phd-tests/runner/src/main.rs line = 167 target = phd_runner
32025-03-10T21:07:36.155ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts/runner-0", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 205, tv_nsec: 922383082 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
42025-03-10T21:07:36.155ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
52025-03-10T21:07:36.155ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = d2d8f8ad449df7e2befb7ee2723a442dd74b9b72 file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
62025-03-10T21:07:36.155ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/d2d8f8ad449df7e2befb7ee2723a442dd74b9b72/crucible-nightly.sha256.txt
72025-03-10T21:07:36.263ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
82025-03-10T21:07:36.752ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/9ee2e1f5f8d7cc3543b6985af36c0df7f81498c5/propolis-server.sha256.txt
92025-03-10T21:07:37.017ZDEBGphd-runner: Created new artifact store from manifest file = phd-tests/framework/src/artifacts/store.rs line = 221 store = Store { local_dir: "/phd-artifacts/runner-1", artifacts: {"alpine": Mutex { data: StoredArtifact { description: Artifact { filename: "alpine.iso", kind: GuestOs(Alpine), source: RemoteServer { sha256: "ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1" }, untar: None }, cached_path: None } }, "ovmf": Mutex { data: StoredArtifact { description: Artifact { filename: "OVMF_CODE.fd", kind: Bootrom, source: Buildomat(BuildomatArtifact { repo: Repo("oxidecomputer/edk2"), series: Series("image_debug"), commit: Commit("6d92acf0a22718dd4175d7c64dbcf7aaec3740bd"), sha256: "29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f" }), untar: None }, cached_path: None } }}, downloader: DownloadConfig { timeout: 600s, buildomat_backoff: ExponentialBackoff { current_interval: 1s, initial_interval: 1s, randomization_factor: 0.5, multiplier: 1.5, max_interval: 60s, start_time: Instant { tv_sec: 206, tv_nsec: 784618598 }, max_elapsed_time: Some(1200s), clock: SystemClock }, remote_server_uris: ["https://oxide-omicron-build.s3.amazonaws.com"] } } target = phd_framework::artifacts::store
102025-03-10T21:07:37.017ZINFOphd-runner: Adding Propolis server from local command file = phd-tests/framework/src/artifacts/store.rs line = 229 propolis_server_cmd = /work/phd-test/propolis-server target = phd_framework::artifacts::store
112025-03-10T21:07:37.017ZINFOphd-runner: Adding crucible-downstairs from Buildomat Git revision commit = d2d8f8ad449df7e2befb7ee2723a442dd74b9b72 file = phd-tests/framework/src/artifacts/store.rs line = 313 target = phd_framework::artifacts::store
122025-03-10T21:07:37.017ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/nightly-image/d2d8f8ad449df7e2befb7ee2723a442dd74b9b72/crucible-nightly.sha256.txt
132025-03-10T21:07:37.122ZINFOphd-runner: Adding 'current' Propolis server from Buildomat Git branch 'master' file = phd-tests/framework/src/artifacts/store.rs line = 250 target = phd_framework::artifacts::store
142025-03-10T21:07:37.543ZDEBGphd-runner: Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/9ee2e1f5f8d7cc3543b6985af36c0df7f81498c5/propolis-server.sha256.txt
152025-03-10T21:07:37.646ZINFOphd-runner: Running 3 test(s) file = phd-tests/runner/src/execute.rs line = 161 target = phd_runner::execute
162025-03-10T21:07:37.647ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
172025-03-10T21:07:37.647ZINFOphd-runner: [EXECUTION_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
182025-03-10T21:07:37.647ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
192025-03-10T21:07:37.647ZINFOphd-runner: [EXECUTION_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 25 target = phd_runner::fixtures
202025-03-10T21:07:37.647ZINFOphd-runner: Starting test phd_tests::migrate::from_base::migration_from_base_and_back file = phd-tests/runner/src/execute.rs line = 101 target = phd_runner::execute
212025-03-10T21:07:37.647ZINFOphd-runner: Starting test phd_tests::migrate::from_base::serial_history file = phd-tests/runner/src/execute.rs line = 101 target = phd_runner::execute
222025-03-10T21:07:37.647ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
232025-03-10T21:07:37.647ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
242025-03-10T21:07:37.647ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
252025-03-10T21:07:37.647ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
262025-03-10T21:07:37.647ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
272025-03-10T21:07:37.647ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
282025-03-10T21:07:37.648ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-0/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store
292025-03-10T21:07:37.648ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-1/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
302025-03-10T21:07:37.648ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
312025-03-10T21:07:37.648ZDEBGphd-runner: [TEST - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/edk2/image_debug/6d92acf0a22718dd4175d7c64dbcf7aaec3740bd/OVMF_CODE.fd
322025-03-10T21:07:38.245ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-1/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
332025-03-10T21:07:38.245ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 621 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store timeout = 600s
342025-03-10T21:07:38.274ZDEBGphd-runner: [TEST - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store
352025-03-10T21:07:38.274ZDEBGphd-runner: [TEST - EVENT] Downloading alpine.iso from https://oxide-omicron-build.s3.amazonaws.com/alpine.iso file = phd-tests/framework/src/artifacts/store.rs line = 621 path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store timeout = 600s
362025-03-10T21:08:12.362ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
372025-03-10T21:08:12.362ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
382025-03-10T21:08:12.405ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 3.90M 1.80T 3.80M /phd-artifacts\n target = phd_framework::zfs
392025-03-10T21:08:12.405ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-f8ef2ae3-f355-46e0-85f8-83bde2919e44"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = snapshot
402025-03-10T21:08:12.462ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-f8ef2ae3-f355-46e0-85f8-83bde2919e44", "phd-artifacts/phd-clone-3bb4d8f0-493e-49cf-9703-8c855f29d4fe"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = clone
412025-03-10T21:08:12.490ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-3bb4d8f0-493e-49cf-9703-8c855f29d4fe"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = list
422025-03-10T21:08:12.517ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::serial_history stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-3bb4d8f0-493e-49cf-9703-8c855f29d4fe 0B 1.80T 50.8M /phd-artifacts/phd-clone-3bb4d8f0-493e-49cf-9703-8c855f29d4fe\n target = phd_framework::zfs
432025-03-10T21:08:12.518ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
442025-03-10T21:08:12.518ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 211 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm_name = migration_serial_history_base
452025-03-10T21:08:12.518ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-0/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server.tar.gz path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store
462025-03-10T21:08:12.518ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/9ee2e1f5f8d7cc3543b6985af36c0df7f81498c5/propolis-server.tar.gz
472025-03-10T21:08:12.552ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-1/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
482025-03-10T21:08:12.552ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/runner-1/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
492025-03-10T21:08:12.574ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 50.9M 1.80T 50.8M /phd-artifacts\n target = phd_framework::zfs
502025-03-10T21:08:12.574ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-68aa1763-4002-4b53-ae10-15f8d1625940"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = snapshot
512025-03-10T21:08:12.633ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-68aa1763-4002-4b53-ae10-15f8d1625940", "phd-artifacts/phd-clone-9d36a2f8-ffed-490c-b405-c06b63013ecc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = clone
522025-03-10T21:08:12.657ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-9d36a2f8-ffed-490c-b405-c06b63013ecc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = list
532025-03-10T21:08:12.671ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::migration_from_base_and_back stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-9d36a2f8-ffed-490c-b405-c06b63013ecc 0B 1.80T 97.9M /phd-artifacts/phd-clone-9d36a2f8-ffed-490c-b405-c06b63013ecc\n target = phd_framework::zfs
542025-03-10T21:08:12.671ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
552025-03-10T21:08:12.671ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 211 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm_name = migration_from_base_and_back
562025-03-10T21:08:12.671ZDEBGphd-runner: [NEW - EVENT] checking for existing copy of artifact file = phd-tests/framework/src/artifacts/store.rs line = 89 maybe_path = /phd-artifacts/runner-1/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server.tar.gz path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
572025-03-10T21:08:12.671ZDEBGphd-runner: [NEW - EVENT] Downloading file from Buildomat... file = phd-tests/framework/src/artifacts/buildomat.rs line = 215 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::buildomat timeout = 600s uri = https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/phd_build/9ee2e1f5f8d7cc3543b6985af36c0df7f81498c5/propolis-server.tar.gz
582025-03-10T21:08:37.044ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/runner-0/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store untar_path = propolis-server
592025-03-10T21:08:37.044ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 521 path = phd_tests::migrate::from_base::serial_history target = phd_framework::artifacts::store
602025-03-10T21:08:37.215ZDEBGphd-runner: [NEW - EVENT] Extracting artifact from tarball extracted_path = /phd-artifacts/runner-1/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server file = phd-tests/framework/src/artifacts/store.rs line = 162 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store untar_path = propolis-server
612025-03-10T21:08:37.215ZDEBGphd-runner: [NEW - EVENT] Extracting gzipped tarball... file = phd-tests/framework/src/artifacts/store.rs line = 521 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::artifacts::store
622025-03-10T21:08:50.461ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/runner-0/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::serial_history server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/runner-0/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server" target = phd_framework::test_vm::server
632025-03-10T21:08:50.461ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::serial_history stderr_path = "/tmp/propolis-phd/runner-0/migration_serial_history_base.stderr.log" stdout_path = "/tmp/propolis-phd/runner-0/migration_serial_history_base.stdout.log" target = phd_framework::server_log_mode
642025-03-10T21:08:50.576ZINFOphd-runner: [NEW - EVENT] Launched server with pid 871 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm::server
652025-03-10T21:08:50.576ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 38057 file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm
662025-03-10T21:08:50.576ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 69048244-dda4-48f2-81d9-16e8db1760ae
672025-03-10T21:08:50.577ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 320 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 69048244-dda4-48f2-81d9-16e8db1760ae
682025-03-10T21:08:50.597ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/runner-1/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::migration_from_base_and_back server_addr = 127.0.0.1:9125 server_path = "/phd-artifacts/runner-1/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server" target = phd_framework::test_vm::server
692025-03-10T21:08:50.597ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::migration_from_base_and_back stderr_path = "/tmp/propolis-phd/runner-1/migration_from_base_and_back.stderr.log" stdout_path = "/tmp/propolis-phd/runner-1/migration_from_base_and_back.stdout.log" target = phd_framework::server_log_mode
702025-03-10T21:08:50.639ZINFOphd-runner: [NEW - EVENT] Launched server with pid 872 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm::server
712025-03-10T21:08:50.639ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 37967 file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm
722025-03-10T21:08:50.639ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = b1850a39-20a7-473a-9111-106afa27f677
732025-03-10T21:08:50.640ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9125/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 320 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = b1850a39-20a7-473a-9111-106afa27f677
742025-03-10T21:08:51.101ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6261907a-2dca-45bb-ba5d-6602517a27ec", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6261907a-2dca-45bb-ba5d-6602517a27ec" } file = phd-tests/framework/src/test_vm/mod.rs line = 324 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 69048244-dda4-48f2-81d9-16e8db1760ae
752025-03-10T21:08:51.102ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 525 file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::serial_history target = phd_framework::test_vm vm = migration_serial_history_base vm_id = 69048244-dda4-48f2-81d9-16e8db1760ae
762025-03-10T21:08:51.102ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::serial_history self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
772025-03-10T21:08:51.157ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
782025-03-10T21:08:51.157ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-3bb4d8f0-493e-49cf-9703-8c855f29d4fe"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
792025-03-10T21:08:51.340ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs
802025-03-10T21:08:51.341ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-f8ef2ae3-f355-46e0-85f8-83bde2919e44"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::serial_history target = phd_framework::zfs verb = destroy
812025-03-10T21:08:51.391ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 73743 file = phd-tests/tests/src/migrate.rs line = 40 path = phd_tests::migrate::from_base::serial_history target = phd_tests::migrate::from_base
822025-03-10T21:08:51.492ZINFOphd-runner: test phd_tests::migrate::from_base::serial_history ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "6261907a-2dca-45bb-ba5d-6602517a27ec", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "6261907a-2dca-45bb-ba5d-6602517a27ec" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:2014:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:336:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:276:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:414:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:402:32 7: phd_tests::migrate::run_serial_history_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:345:21 8: phd_tests::migrate::from_base::serial_history::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:46:10 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::serial_history::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:40:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::run_tests::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:117:58 14: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 15: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 16: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 17: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 18: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 19: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 20: __rust_try 21: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 22: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 23: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 24: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 25: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 26: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 27: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 28: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:436:9 29: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:661:17 30: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:107:5 31: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:73:5 32: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:597:9 33: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:548:24 34: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:513:21 35: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9 36: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:26 37: std::thread::local::LocalKey<T>::try_with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 38: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 39: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:9 40: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:508:9 41: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16 42: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:500:5 43: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:466:45 44: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/task.rs:42:21 45: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 46: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 47: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 48: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 49: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 50: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 51: __rust_try 52: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 53: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 54: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 55: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 56: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 57: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 58: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 59: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:473:9 60: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:160:9 61: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:518:17 62: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:476:13 63: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/backtrace.rs:152:18 64: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:564:17 65: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 66: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 67: __rust_try 68: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 69: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 70: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:562:30 71: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 72: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 74: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/pal/unix/thread.rs:106:17 75: _thrp_setup 76: <unknown> file = phd-tests/runner/src/execute.rs line = 119 target = phd_runner::execute
832025-03-10T21:08:51.492ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
842025-03-10T21:08:51.492ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
852025-03-10T21:08:51.493ZINFOphd-runner: Starting test phd_tests::migrate::from_base::can_migrate_from_base file = phd-tests/runner/src/execute.rs line = 101 target = phd_runner::execute
862025-03-10T21:08:51.493ZINFOphd-runner: [TEST_SETUP - START] file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
872025-03-10T21:08:51.493ZINFOphd-runner: [TEST_SETUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 41 target = phd_runner::fixtures
882025-03-10T21:08:51.493ZINFOphd-runner: [TEST - START] file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
892025-03-10T21:08:51.493ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-0/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd target = phd_framework::artifacts::store
902025-03-10T21:08:51.493ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
912025-03-10T21:08:51.493ZDEBGphd-runner: [TEST - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso target = phd_framework::artifacts::store
922025-03-10T21:08:51.493ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["/phd-artifacts/runner-0/ba8007f74f9b54fbae3b2520da577831b4834778a498d732f091260c61aa7ca1/alpine.iso"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
932025-03-10T21:08:51.543ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts 1.34G 1.80T 1.34G /phd-artifacts\n target = phd_framework::zfs
942025-03-10T21:08:51.543ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-1c3f9a47-443c-4e05-8e70-ef38dfc3fa6e"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = snapshot
952025-03-10T21:08:51.584ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-1c3f9a47-443c-4e05-8e70-ef38dfc3fa6e", "phd-artifacts/phd-clone-e2dc39fc-3c98-41a4-b6ba-a24d44613e82"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = clone
962025-03-10T21:08:51.623ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-e2dc39fc-3c98-41a4-b6ba-a24d44613e82"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = list
972025-03-10T21:08:51.663ZDEBGphd-runner: [TEST - EVENT] parsing zfs list output file = phd-tests/framework/src/zfs.rs line = 238 path = phd_tests::migrate::from_base::can_migrate_from_base stdout = NAME USED AVAIL REFER MOUNTPOINT\nphd-artifacts/phd-clone-e2dc39fc-3c98-41a4-b6ba-a24d44613e82 0B 1.80T 1.34G /phd-artifacts/phd-clone-e2dc39fc-3c98-41a4-b6ba-a24d44613e82\n target = phd_framework::zfs
982025-03-10T21:08:51.663ZINFOphd-runner: [NEW - START] file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
992025-03-10T21:08:51.663ZINFOphd-runner: [NEW - EVENT] phd_framework::test_vm environment = EnvironmentSpec { location: Local, propolis_artifact: "__BASE_PROPOLIS" } file = phd-tests/framework/src/test_vm/mod.rs guest_os_kind = Alpine line = 211 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm_name = migration_from_base
1002025-03-10T21:08:51.663ZDEBGphd-runner: [NEW - EVENT] Verified artifact already exists file = phd-tests/framework/src/artifacts/store.rs line = 45 path = /phd-artifacts/runner-0/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server target = phd_framework::artifacts::store
1012025-03-10T21:08:51.663ZINFOphd-runner: [NEW - EVENT] Launching Propolis server bootrom_path = "/phd-artifacts/runner-0/29813374b58e3b77fb665f2d95cb3bab37d44fdd2c4fce2a70de9d76a3512a4f/OVMF_CODE.fd" file = phd-tests/framework/src/test_vm/server.rs line = 55 path = phd_tests::migrate::from_base::can_migrate_from_base server_addr = 127.0.0.1:9000 server_path = "/phd-artifacts/runner-0/699c2d2c5e97e63733ab85fce044388baefe882761373e5c02190990dd286770/propolis-server" target = phd_framework::test_vm::server
1022025-03-10T21:08:51.663ZINFOphd-runner: [NEW - EVENT] Opening server log files file = phd-tests/framework/src/server_log_mode.rs line = 64 path = phd_tests::migrate::from_base::can_migrate_from_base stderr_path = "/tmp/propolis-phd/runner-0/migration_from_base.stderr.log" stdout_path = "/tmp/propolis-phd/runner-0/migration_from_base.stdout.log" target = phd_framework::server_log_mode
1032025-03-10T21:08:51.696ZINFOphd-runner: [NEW - EVENT] Launched server with pid 880 file = phd-tests/framework/src/test_vm/server.rs line = 107 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm::server
1042025-03-10T21:08:51.697ZINFOphd-runner: [NEW - END] elapsed_milliseconds = 33 file = phd-tests/framework/src/test_vm/mod.rs line = 193 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm
1052025-03-10T21:08:51.697ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - START] file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = cadd903e-2784-4c2d-b4d0-e1f60ff9ed27
1062025-03-10T21:08:51.697ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] retriable error from instance_spec_ensure e = Communication Error: error sending request for url (http://127.0.0.1:9000/instance) file = phd-tests/framework/src/test_vm/mod.rs line = 320 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = cadd903e-2784-4c2d-b4d0-e1f60ff9ed27
1072025-03-10T21:08:51.718ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "dfff96e5-efaf-4183-95a5-953338c065d9", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "dfff96e5-efaf-4183-95a5-953338c065d9" } file = phd-tests/framework/src/test_vm/mod.rs line = 324 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = b1850a39-20a7-473a-9111-106afa27f677
1082025-03-10T21:08:51.718ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 1078 file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::test_vm vm = migration_from_base_and_back vm_id = b1850a39-20a7-473a-9111-106afa27f677
1092025-03-10T21:08:51.718ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::migration_from_base_and_back self.address = 127.0.0.1:9125 target = phd_framework::test_vm::server
1102025-03-10T21:08:51.760ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
1112025-03-10T21:08:51.761ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-9d36a2f8-ffed-490c-b405-c06b63013ecc"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
1122025-03-10T21:08:51.811ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs
1132025-03-10T21:08:51.811ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-68aa1763-4002-4b53-ae10-15f8d1625940"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_framework::zfs verb = destroy
1142025-03-10T21:08:51.854ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 74206 file = phd-tests/tests/src/migrate.rs line = 52 path = phd_tests::migrate::from_base::migration_from_base_and_back target = phd_tests::migrate::from_base
1152025-03-10T21:08:51.855ZINFOphd-runner: test phd_tests::migrate::from_base::migration_from_base_and_back ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "dfff96e5-efaf-4183-95a5-953338c065d9", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:51 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "dfff96e5-efaf-4183-95a5-953338c065d9" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:2014:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:336:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:276:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:414:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:402:32 7: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:56:25 8: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 9: phd_tests::migrate::from_base::migration_from_base_and_back::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:52:5 10: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 11: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 12: phd_runner::execute::run_tests_with_ctx::{{closure}}::run_tests::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:117:58 13: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 14: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 15: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 16: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 17: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 18: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 19: __rust_try 20: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 21: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 22: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 23: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 24: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 25: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 26: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 27: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:436:9 28: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:661:17 29: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:107:5 30: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:73:5 31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:597:9 32: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:548:24 33: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:513:21 34: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9 35: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:26 36: std::thread::local::LocalKey<T>::try_with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 37: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 38: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:9 39: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:508:9 40: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16 41: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:500:5 42: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:466:45 43: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/task.rs:42:21 44: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 45: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 46: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 47: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 48: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 49: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 50: __rust_try 51: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 52: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 53: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 54: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 55: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 56: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 57: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 58: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:473:9 59: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:160:9 60: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:518:17 61: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:476:13 62: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/backtrace.rs:152:18 63: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:564:17 64: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 65: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 66: __rust_try 67: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 68: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 69: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:562:30 70: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 71: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 72: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 73: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/pal/unix/thread.rs:106:17 74: _thrp_setup 75: <unknown> file = phd-tests/runner/src/execute.rs line = 119 target = phd_runner::execute
1162025-03-10T21:08:51.855ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1172025-03-10T21:08:51.855ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1182025-03-10T21:08:51.855ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1192025-03-10T21:08:51.855ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1202025-03-10T21:08:52.088ZERROphd-runner: [INSTANCE_ENSURE_INTERNAL - EVENT] permanent error from instance_spec_ensure e = Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "571e4e3d-1a71-4701-8991-5958d651cb01", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "571e4e3d-1a71-4701-8991-5958d651cb01" } file = phd-tests/framework/src/test_vm/mod.rs line = 324 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = cadd903e-2784-4c2d-b4d0-e1f60ff9ed27
1212025-03-10T21:08:52.088ZINFOphd-runner: [INSTANCE_ENSURE_INTERNAL - END] elapsed_milliseconds = 391 file = phd-tests/framework/src/test_vm/mod.rs line = 276 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::test_vm vm = migration_from_base vm_id = cadd903e-2784-4c2d-b4d0-e1f60ff9ed27
1222025-03-10T21:08:52.088ZDEBGphd-runner: [TEST - EVENT] Killing Propolis server process file = phd-tests/framework/src/test_vm/server.rs line = 125 path = phd_tests::migrate::from_base::can_migrate_from_base self.address = 127.0.0.1:9000 target = phd_framework::test_vm::server
1232025-03-10T21:08:52.137ZDEBGphd-runner: [TEST - EVENT] zfs clone dropped file = phd-tests/framework/src/zfs.rs line = 83 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1242025-03-10T21:08:52.137ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts/phd-clone-e2dc39fc-3c98-41a4-b6ba-a24d44613e82"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1252025-03-10T21:08:52.192ZDEBGphd-runner: [TEST - EVENT] zfs snapshot dropped file = phd-tests/framework/src/zfs.rs line = 57 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs
1262025-03-10T21:08:52.192ZDEBGphd-runner: [TEST - EVENT] executing ZFS command args = ["phd-artifacts@phd-1c3f9a47-443c-4e05-8e70-ef38dfc3fa6e"] file = phd-tests/framework/src/zfs.rs line = 304 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_framework::zfs verb = destroy
1272025-03-10T21:08:52.240ZINFOphd-runner: [TEST - END] elapsed_milliseconds = 747 file = phd-tests/tests/src/migrate.rs line = 34 path = phd_tests::migrate::from_base::can_migrate_from_base target = phd_tests::migrate::from_base
1282025-03-10T21:08:52.241ZINFOphd-runner: test phd_tests::migrate::from_base::can_migrate_from_base ... FAILED: Error Response: status: 500 Internal Server Error; headers: {"content-type": "application/json", "x-request-id": "571e4e3d-1a71-4701-8991-5958d651cb01", "content-length": "124", "date": "Mon, 10 Mar 2025 21:08:52 GMT"}; value: Error { error_code: Some("Internal"), message: "Internal Server Error", request_id: "571e4e3d-1a71-4701-8991-5958d651cb01" } Stack backtrace: 0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.89/src/backtrace.rs:27:14 1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:2014:27 2: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:336:9 3: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 4: phd_framework::test_vm::TestVm::instance_ensure_internal::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:276:5 5: phd_framework::test_vm::TestVm::instance_ensure::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:414:22 6: phd_framework::test_vm::TestVm::launch::{{closure}} at ./oxidecomputer/propolis/phd-tests/framework/src/test_vm/mod.rs:402:32 7: phd_tests::migrate::run_smoke_test::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:315:21 8: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}}::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:37:14 9: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tracing-0.1.40/src/instrument.rs:321:9 10: phd_tests::migrate::from_base::can_migrate_from_base::{{closure}} at ./oxidecomputer/propolis/phd-tests/tests/src/migrate.rs:34:5 11: <core::pin::Pin<P> as core::future::future::Future>::poll at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 12: phd_testcase::TestCase::run::{{closure}} at ./oxidecomputer/propolis/phd-tests/testcase/src/lib.rs:79:32 13: phd_runner::execute::run_tests_with_ctx::{{closure}}::run_tests::{{closure}} at ./oxidecomputer/propolis/phd-tests/runner/src/execute.rs:117:58 14: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 15: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 16: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 17: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 18: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 19: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 20: __rust_try 21: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 22: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 23: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 24: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 25: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 26: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 27: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 28: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:436:9 29: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:661:17 30: tokio::runtime::coop::with_budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:107:5 31: tokio::runtime::coop::budget at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/coop.rs:73:5 32: tokio::runtime::scheduler::multi_thread::worker::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:597:9 33: tokio::runtime::scheduler::multi_thread::worker::Context::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:548:24 34: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:513:21 35: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9 36: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:26 37: std::thread::local::LocalKey<T>::try_with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 38: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 39: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context.rs:180:9 40: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:508:9 41: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16 42: tokio::runtime::scheduler::multi_thread::worker::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:500:5 43: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/scheduler/multi_thread/worker.rs:466:45 44: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/task.rs:42:21 45: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:331:17 46: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/loom/std/unsafe_cell.rs:16:9 47: tokio::runtime::task::core::Core<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/core.rs:320:13 48: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:500:19 49: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 50: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 51: __rust_try 52: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 53: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 54: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:488:18 55: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:209:27 56: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/harness.rs:154:15 57: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:271:5 58: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/raw.rs:201:18 59: tokio::runtime::task::UnownedTask<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/task/mod.rs:473:9 60: tokio::runtime::blocking::pool::Task::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:160:9 61: tokio::runtime::blocking::pool::Inner::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:518:17 62: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.40.0/src/runtime/blocking/pool.rs:476:13 63: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/backtrace.rs:152:18 64: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:564:17 65: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 66: std::panicking::try::do_call at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 67: __rust_try 68: std::panicking::try at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 69: std::panic::catch_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 70: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/mod.rs:562:30 71: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 72: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/alloc/src/boxed.rs:1993:9 74: std::sys::pal::unix::thread::Thread::new::thread_start at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/sys/pal/unix/thread.rs:106:17 75: _thrp_setup 76: <unknown> file = phd-tests/runner/src/execute.rs line = 119 target = phd_runner::execute
1292025-03-10T21:08:52.241ZINFOphd-runner: [TEST_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1302025-03-10T21:08:52.241ZINFOphd-runner: [TEST_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 51 target = phd_runner::fixtures
1312025-03-10T21:08:52.241ZINFOphd-runner: [EXECUTION_CLEANUP - START] file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
1322025-03-10T21:08:52.242ZINFOphd-runner: [EXECUTION_CLEANUP - END] elapsed_milliseconds = 0 file = phd-tests/runner/src/fixtures.rs line = 35 target = phd_runner::fixtures
133 
134 failures:
135 phd_tests::migrate::from_base::serial_history
136 phd_tests::migrate::from_base::migration_from_base_and_back
137 phd_tests::migrate::from_base::can_migrate_from_base
138 
139 test result: FAILED. 0 passed; 3 failed; 0 skipped; 0 not run; finished in 74.60s
140 
1412025-03-10T21:08:52.242ZDEBGphd-runner: phd_runner exit_code = 3 file = phd-tests/runner/src/main.rs line = 41 target = phd_runner