(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'integration_tests::pantry::test_cannot_finalize_attached_disk' panicked at clients/sled-agent-client/src/lib.rs:379:14: instance_finish_transition() failed unexpectedly: reqwest::Error { kind: Request, url: "http://[::1]:39722/vmms/66f85e20-04e6-4bcc-a690-b1171ee62c8b/poke", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(IncompleteMessage)) } stack backtrace: 0: rust_begin_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5 1: core::panicking::panic_fmt at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1704:5 3: core::result::Result<T,E>::expect at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1061:23 4: {async_block#0} at /work/oxidecomputer/omicron/clients/sled-agent-client/src/lib.rs:375:9 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 6: {async_fn#0} at ./tests/integration_tests/pantry.rs:96:62 7: {async_fn#0} at ./tests/integration_tests/pantry.rs:161:44 8: {async_fn#0} at ./tests/integration_tests/pantry.rs:845:74 9: {async_block#0} at ./tests/integration_tests/pantry.rs:823:1 10: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 11: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:57 13: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:107:5 14: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:73:5 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:25 16: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:428:19 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:728:36 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:68 19: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/scoped.rs:40:9 20: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:26 21: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 22: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 23: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:9 24: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:27 25: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:716:19 26: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:196:28 27: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:184:9 29: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:368:47 30: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:342:13 31: test_cannot_finalize_attached_disk at ./tests/integration_tests/pantry.rs:823:1 32: test_all::integration_tests::pantry::test_cannot_finalize_attached_disk::{{closure}} at ./tests/integration_tests/pantry.rs:826:2 33: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 34: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp4VB9I8" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp4VB9I8/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 24276) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.1-clickhouse-3RKhE6) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.1-clickhouse-3RKhE6: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDju6H9 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpIqbPr0 running 1 test sled b6d65341-167c-41df-9b5c-41cded99c229 successfully installed routes ResolvedVpcRouteSet { id: RouterId { vni: Vni(15600428), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }, version: None, routes: {} } sled b6d65341-167c-41df-9b5c-41cded99c229 successfully installed routes ResolvedVpcRouteSet { id: RouterId { vni: Vni(15600428), kind: System }, version: Some(RouterVersion { router_id: 9626c381-85e4-4148-8ec0-7b3feee129c4, version: 3 }), routes: {ResolvedVpcRoute { dest: V6(Ipv6Net { addr: fdc8:9fc9:4af8::, width: 64 }), target: VpcSubnet(V6(Ipv6Net { addr: fdc8:9fc9:4af8::, width: 64 })) }, ResolvedVpcRoute { dest: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), target: VpcSubnet(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }, ResolvedVpcRoute { dest: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), target: InternetGateway(Instance(2e7d5fce-108d-4db9-81b2-8352b97a4a43)) }, ResolvedVpcRoute { dest: V6(Ipv6Net { addr: ::, width: 0 }), target: InternetGateway(Instance(2e7d5fce-108d-4db9-81b2-8352b97a4a43)) }} } test integration_tests::pantry::test_cannot_finalize_attached_disk ... FAILED failures: failures: integration_tests::pantry::test_cannot_finalize_attached_disk test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 425 filtered out; finished in 37.90s log file: /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.0.log" DB URL: postgresql://root@[::1]:35460/omicron?sslmode=disable DB address: [::1]:35460 log file: /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.2.log" thread 'integration_tests::pantry::test_cannot_finalize_attached_disk' panicked at sled-agent/src/sim/collection.rs:274:65: called `Result::unwrap()` on an `Err` value: InternalError { internal_message: "Communication Error: error sending request for url (http://127.0.0.1:53359/vmms/66f85e20-04e6-4bcc-a690-b1171ee62c8b)" } stack backtrace: 0: rust_begin_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5 1: core::panicking::panic_fmt at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1704:5 3: unwrap<(), omicron_common::api::external::error::Error> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1109:23 4: {async_fn#0}<omicron_sled_agent::sim::instance::SimInstance> at /work/oxidecomputer/omicron/sled-agent/src/sim/collection.rs:274:13 5: {async_fn#0} at /work/oxidecomputer/omicron/sled-agent/src/sim/sled_agent.rs:528:58 6: {async_fn#0} at /work/oxidecomputer/omicron/sled-agent/src/sim/http_entrypoints.rs:773:38 7: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, dropshot::handler::HttpResponseUpdatedNoContent, dropshot::error::HttpError, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/handler.rs:664:78 8: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 9: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>, fn(dropshot::handler::RequestContext<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>) -> omicron_sled_agent::sim::http_entrypoints::{impl#1}::from::instance_poke_post::{async_fn_env#0}, (dropshot::extractor::path::Path<sled_agent_api::VmmPathParam>), dropshot::handler::HttpResponseUpdatedNoContent> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/handler.rs:785:16 10: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=core::result::Result<http::response::Response<dropshot::body::Body>, dropshot::handler::HandlerError>> + core::marker::Send), alloc::alloc::Global>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 11: {async_block#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.0/src/server.rs:937:69 12: tokio::runtime::task::core::Core<T,S>::poll::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/core.rs:331:17 13: with_mut<tokio::runtime::task::core::Stage<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/loom/std/unsafe_cell.rs:16:9 14: poll<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/core.rs:320:13 15: tokio::runtime::task::harness::poll_future::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:532:19 16: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panic/unwind_safe.rs:272:9 17: do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:584:40 18: __rust_try 19: try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:547:19 20: catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<dropshot::server::http_request_handle::{async_fn#0}::{async_block_env#0}<alloc::sync::Arc<omicron_sled_agent::sim::sled_agent::SledAgent, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panic.rs:358:14 21: tokio::runtime::task::harness::poll_future at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:520:18 22: tokio::runtime::task::harness::Harness<T,S>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:209:27 23: tokio::runtime::task::harness::Harness<T,S>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/harness.rs:154:15 24: tokio::runtime::task::raw::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/raw.rs:271:5 25: tokio::runtime::task::raw::RawTask::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/raw.rs:201:18 26: tokio::runtime::task::LocalNotified<S>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/task/mod.rs:449:9 27: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:770:25 28: with_budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:107:5 29: budget<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:73:5 30: {closure#0}<(), tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:364:43 31: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:428:19 32: tokio::runtime::scheduler::current_thread::Context::run_task at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:364:23 33: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:769:35 34: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:68 35: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/scoped.rs:40:9 36: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:26 37: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 38: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 39: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:9 40: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:27 41: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:716:19 42: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:196:28 43: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16 44: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:184:9 45: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:368:47 46: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:342:13 47: test_cannot_finalize_attached_disk at ./tests/integration_tests/pantry.rs:823:1 48: test_all::integration_tests::pantry::test_cannot_finalize_attached_disk::{{closure}} at ./tests/integration_tests/pantry.rs:826:2 49: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 50: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. thread 'integration_tests::pantry::test_cannot_finalize_attached_disk' panicked at clients/sled-agent-client/src/lib.rs:379:14: instance_finish_transition() failed unexpectedly: reqwest::Error { kind: Request, url: "http://[::1]:39722/vmms/66f85e20-04e6-4bcc-a690-b1171ee62c8b/poke", source: hyper_util::client::legacy::Error(SendRequest, hyper::Error(IncompleteMessage)) } stack backtrace: 0: rust_begin_unwind at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/panicking.rs:692:5 1: core::panicking::panic_fmt at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1704:5 3: core::result::Result<T,E>::expect at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/result.rs:1061:23 4: {async_block#0} at /work/oxidecomputer/omicron/clients/sled-agent-client/src/lib.rs:375:9 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 6: {async_fn#0} at ./tests/integration_tests/pantry.rs:96:62 7: {async_fn#0} at ./tests/integration_tests/pantry.rs:161:44 8: {async_fn#0} at ./tests/integration_tests/pantry.rs:845:74 9: {async_block#0} at ./tests/integration_tests/pantry.rs:823:1 10: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 11: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/future/future.rs:124:9 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:57 13: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:107:5 14: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/coop.rs:73:5 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:729:25 16: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:428:19 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:728:36 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:68 19: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/scoped.rs:40:9 20: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:26 21: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:308:12 22: std::thread::local::LocalKey<T>::with at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/std/src/thread/local.rs:272:9 23: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context.rs:180:9 24: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:807:27 25: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:716:19 26: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:196:28 27: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/context/runtime.rs:65:16 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/scheduler/current_thread/mod.rs:184:9 29: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:368:47 30: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.43.0/src/runtime/runtime.rs:342:13 31: test_cannot_finalize_attached_disk at ./tests/integration_tests/pantry.rs:823:1 32: test_all::integration_tests::pantry::test_cannot_finalize_attached_disk::{{closure}} at ./tests/integration_tests/pantry.rs:826:2 33: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 34: core::ops::function::FnOnce::call_once at /rustc/4d91de4e48198da2e33413efdcd9cd2cc0c46688/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp4VB9I8" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp4VB9I8/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 24276) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.1-clickhouse-3RKhE6) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-e287ef60b0239288-test_cannot_finalize_attached_disk.24248.1-clickhouse-3RKhE6: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDju6H9 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpIqbPr0