(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind' panicked at clients/sled-agent-client/src/lib.rs:369:14: instance_finish_transition() failed unexpectedly: reqwest::Error { kind: Request, url: "http://[::1]:33095/vmms/d7e136b0-d474-4210-a27b-da80cb782ba7/poke", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(IncompleteMessage)) } stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:697:5 1: core::panicking::panic_fmt at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1732:5 3: core::result::Result<T,E>::expect at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1089:23 4: {async_block#0} at /work/oxidecomputer/omicron/clients/sled-agent-client/src/lib.rs:365:9 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 6: {async_fn#0} at ./src/app/sagas/test_helpers.rs:141:38 7: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2452:70 8: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2332:51 9: {async_block#0} at ./src/app/sagas/instance_update/mod.rs:2367:30 10: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=omicron_nexus::app::sagas::instance_update::RealParams> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 11: {async_fn#0}<omicron_nexus::app::sagas::instance_update::SagaDoActualInstanceUpdate, omicron_nexus::app::sagas::instance_update::test::{impl#0}::run_unwinding_test::{async_fn#0}::{closure_env#0}, omicron_nexus::app::sagas::instance_update::test::{impl#0}::run_unwinding_test::{async_fn#0}::{closure_env#1}> at ./src/app/sagas/test_helpers.rs:597:36 12: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2394:14 13: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2217:14 14: {async_block#0} at ./src/app/sagas/instance_update/mod.rs:2209:5 15: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 16: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:54 18: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:167:5 19: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:133:5 20: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:25 21: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:432:19 22: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:741:36 23: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:68 24: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/scoped.rs:40:9 25: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:26 26: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:315:12 27: std::thread::local::LocalKey<T>::with at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:279:15 28: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:9 29: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:27 30: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:729:19 31: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:200:28 32: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/runtime.rs:65:16 33: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:188:9 34: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:356:47 35: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:330:13 36: test_migration_source_failed_destroyed_can_unwind at ./src/app/sagas/instance_update/mod.rs:2209:5 37: omicron_nexus::app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind::{{closure}} at ./src/app/sagas/instance_update/mod.rs:2212:6 38: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 39: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp5d2VfP" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp5d2VfP/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 95047) and a temporary directory leaked, /var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.1-clickhouse-or04Ll) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDHa75c WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpVdmYy9 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpZhRryC WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpSJD7aY running 1 test test app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind ... FAILED failures: failures: app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 174 filtered out; finished in 37.71s log file: /var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.0.log note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.0.log" DB URL: postgresql://root@[::1]:42535/omicron?sslmode=disable DB address: [::1]:42535 log file: /var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.2.log note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.2.log" log file: /var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.3.log note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.3.log" thread 'app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind' panicked at sled-agent/src/sim/collection.rs:274:65: called `Result::unwrap()` on an `Err` value: InternalError { internal_message: "Communication Error: error sending request for url (http://127.0.0.1:42165/vmms/d7e136b0-d474-4210-a27b-da80cb782ba7)" } stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:697:5 1: core::panicking::panic_fmt at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1732:5 3: unwrap<(), omicron_common::api::external::error::Error> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1137:23 4: {async_fn#0}<omicron_sled_agent::sim::instance::SimInstance> at /work/oxidecomputer/omicron/sled-agent/src/sim/collection.rs:274:13 5: {async_fn#0} at /work/oxidecomputer/omicron/sled-agent/src/sim/sled_agent.rs:523:58 6: {async_fn#0} at /work/oxidecomputer/omicron/sled-agent/src/sim/http_entrypoints.rs:823:38 7: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, dropshot::handler::HttpResponseUpdatedNoContent, dropshot::error::HttpError, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/handler.rs:664:78 8: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 9: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, (dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>), dropshot::handler::HttpResponseUpdatedNoContent> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/handler.rs:785:16 10: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 11: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:937:69 12: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/core.rs:365:17 13: with_mut<tokio::runtime::task::core::Stage<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/loom/std/unsafe_cell.rs:16:9 14: poll<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/core.rs:354:13 15: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/harness.rs:535:19 16: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panic/unwind_safe.rs:272:9 17: do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:589:40 18: __rust_try 19: try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:552:19 20: catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panic.rs:359:14 21: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/harness.rs:523:18 22: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/harness.rs:210:27 23: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/harness.rs:155:15 24: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/raw.rs:325:5 25: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/raw.rs:255:18 26: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/task/mod.rs:509:9 27: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:789:25 28: with_budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:167:5 29: budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:133:5 30: {closure#0}<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:368:43 31: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:432:19 32: tokio::runtime::scheduler::current_thread::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:368:23 33: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:785:35 34: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:68 35: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/scoped.rs:40:9 36: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:26 37: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:315:12 38: std::thread::local::LocalKey<T>::with at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:279:15 39: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:9 40: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:27 41: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:729:19 42: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:200:28 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/runtime.rs:65:16 44: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:188:9 45: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:356:47 46: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:330:13 47: test_migration_source_failed_destroyed_can_unwind at ./src/app/sagas/instance_update/mod.rs:2209:5 48: omicron_nexus::app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind::{{closure}} at ./src/app/sagas/instance_update/mod.rs:2212:6 49: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 50: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. thread 'app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind' panicked at clients/sled-agent-client/src/lib.rs:369:14: instance_finish_transition() failed unexpectedly: reqwest::Error { kind: Request, url: "http://[::1]:33095/vmms/d7e136b0-d474-4210-a27b-da80cb782ba7/poke", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(IncompleteMessage)) } stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:697:5 1: core::panicking::panic_fmt at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1732:5 3: core::result::Result<T,E>::expect at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/result.rs:1089:23 4: {async_block#0} at /work/oxidecomputer/omicron/clients/sled-agent-client/src/lib.rs:365:9 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 6: {async_fn#0} at ./src/app/sagas/test_helpers.rs:141:38 7: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2452:70 8: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2332:51 9: {async_block#0} at ./src/app/sagas/instance_update/mod.rs:2367:30 10: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=omicron_nexus::app::sagas::instance_update::RealParams> + core::marker::Send), alloc::alloc::Global>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 11: {async_fn#0}<omicron_nexus::app::sagas::instance_update::SagaDoActualInstanceUpdate, omicron_nexus::app::sagas::instance_update::test::{impl#0}::run_unwinding_test::{async_fn#0}::{closure_env#0}, omicron_nexus::app::sagas::instance_update::test::{impl#0}::run_unwinding_test::{async_fn#0}::{closure_env#1}> at ./src/app/sagas/test_helpers.rs:597:36 12: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2394:14 13: {async_fn#0} at ./src/app/sagas/instance_update/mod.rs:2217:14 14: {async_block#0} at ./src/app/sagas/instance_update/mod.rs:2209:5 15: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 16: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:54 18: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:167:5 19: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:133:5 20: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:25 21: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:432:19 22: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:741:36 23: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:68 24: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/scoped.rs:40:9 25: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:26 26: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:315:12 27: std::thread::local::LocalKey<T>::with at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:279:15 28: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:9 29: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:27 30: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:729:19 31: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:200:28 32: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/runtime.rs:65:16 33: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:188:9 34: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:356:47 35: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:330:13 36: test_migration_source_failed_destroyed_can_unwind at ./src/app/sagas/instance_update/mod.rs:2209:5 37: omicron_nexus::app::sagas::instance_update::test::test_migration_source_failed_destroyed_can_unwind::{{closure}} at ./src/app/sagas/instance_update/mod.rs:2212:6 38: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 39: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp5d2VfP" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp5d2VfP/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 95047) and a temporary directory leaked, /var/tmp/omicron_tmp/omicron_nexus-7d8c6d7d4c95a475-test_migration_source_failed_destroyed_can_unwind.94910.1-clickhouse-or04Ll) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDHa75c WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpVdmYy9 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpZhRryC WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpSJD7aY