(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'integration_tests::affinity::test_affinity_group_usage' (2) panicked at gateway-test-utils/src/setup.rs:261:6: called `Result::unwrap()` on an `Err` value: TimedOut(30.001472634s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: unwrap<(), omicron_test_utils::dev::poll::Error<core::convert::Infallible>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1233:23 4: {async_fn#0} at /work/oxidecomputer/omicron/gateway-test-utils/src/setup.rs:261:6 5: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:417:10 6: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 7: poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/time/timeout.rs:202:42 8: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:233:59 9: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:1691:14 10: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/nexus_test.rs:94:10 11: {async_block#0} at ./tests/integration_tests/affinity.rs:448:1 12: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 13: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 15: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 16: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 18: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 19: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 20: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 21: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 22: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 23: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 24: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 25: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 26: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 27: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 28: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 29: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 30: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 31: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 32: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 33: test_affinity_group_usage at ./tests/integration_tests/affinity.rs:448:1 34: test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} at ./tests/integration_tests/affinity.rs:449:72 35: <test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 36: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp6VTNZA" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp6VTNZA/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 17245) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.1-clickhouse-IG1AmG) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.1-clickhouse-IG1AmG: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpTba8LB WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpALb69d running 1 test test integration_tests::affinity::test_affinity_group_usage ... FAILED failures: failures: integration_tests::affinity::test_affinity_group_usage test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 38.04s log file: /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.0.log" DB URL: postgresql://root@[::1]:56137/omicron?sslmode=disable DB address: [::1]:56137 log file: /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.2.log" log file: /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.3.log" thread 'integration_tests::affinity::test_affinity_group_usage' (2) panicked at sp-sim/src/gimlet.rs:429:62: called `Result::unwrap()` on an `Err` value: couldn't understand ereport header from [::1]:47491: Size(SizeError) Stack backtrace: 0: <anyhow::Error>::msg::<alloc::string::String> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:690:13 2: sp_sim::ereport::recv_request::{closure#0}::{closure#0} at /work/oxidecomputer/omicron/sp-sim/src/ereport.rs:51:17 3: <core::result::Result<gateway_ereport_messages::Request, zerocopy::error::ConvertError<core::convert::Infallible, zerocopy::error::SizeError<&[u8], gateway_ereport_messages::Request>, zerocopy::error::ValidityError<&[u8], gateway_ereport_messages::Request>>>>::map_err::<anyhow::Error, sp_sim::ereport::recv_request::{closure#0}::{closure#0}> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:968:27 4: {async_fn#0} at /work/oxidecomputer/omicron/sp-sim/src/ereport.rs:50:60 5: <sp_sim::gimlet::UdpTask>::run::{closure#0}::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/macros/select.rs:705:49 6: <core::future::poll_fn::PollFn<<sp_sim::gimlet::UdpTask>::run::{closure#0}::{closure#0}> as core::future::future::Future>::poll at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/poll_fn.rs:151:9 7: {async_fn#0} at /work/oxidecomputer/omicron/sp-sim/src/gimlet.rs:723:13 8: {async_block#6} at /work/oxidecomputer/omicron/sp-sim/src/gimlet.rs:429:56 9: poll<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: <tokio::runtime::task::core::Core<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/core.rs:375:24 11: with_mut<tokio::runtime::task::core::Stage<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/loom/std/unsafe_cell.rs:16:9 12: poll<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/core.rs:364:30 13: tokio::runtime::task::harness::poll_future::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:535:30 14: <core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>::{closure#0}> as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panic/unwind_safe.rs:274:9 15: do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:581:40 16: __rust_try 17: catch_unwind<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:544:19 18: catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panic.rs:359:14 19: tokio::runtime::task::harness::poll_future::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:523:18 20: <tokio::runtime::task::harness::Harness<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:210:27 21: <tokio::runtime::task::harness::Harness<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:155:20 22: tokio::runtime::task::raw::poll::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/raw.rs:337:13 23: <tokio::runtime::task::raw::RawTask>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/raw.rs:267:18 24: <tokio::runtime::task::LocalNotified<alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/mod.rs:515:13 25: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:825:30 26: with_budget<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 27: budget<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 28: {closure#0}<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:373:43 29: <tokio::runtime::scheduler::current_thread::Context>::enter::<(), <tokio::runtime::scheduler::current_thread::Context>::run_task<(), <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1}>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 30: <tokio::runtime::scheduler::current_thread::Context>::run_task::<(), <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:373:28 31: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:821:43 32: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 33: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 34: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 35: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 36: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 37: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 38: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 39: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 40: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 41: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 42: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 43: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 44: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 45: test_affinity_group_usage at ./tests/integration_tests/affinity.rs:448:1 46: test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} at ./tests/integration_tests/affinity.rs:449:72 47: <test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 48: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 49: test::__rust_begin_short_backtrace at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/test/src/lib.rs:663:18 50: test::run_test_in_process::{{closure}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/test/src/lib.rs:686:74 51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panic/unwind_safe.rs:274:9 52: std::panicking::catch_unwind::do_call at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:581:40 53: std::panicking::catch_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:544:19 54: std::panic::catch_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panic.rs:359:14 55: test::run_test_in_process at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/test/src/lib.rs:686:27 56: test::run_test::{{closure}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/test/src/lib.rs:607:43 57: test::run_test::{{closure}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/test/src/lib.rs:637:41 58: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/sys/backtrace.rs:166:18 59: std::thread::lifecycle::spawn_unchecked::{{closure}}::{{closure}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/lifecycle.rs:91:13 60: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panic/unwind_safe.rs:274:9 61: std::panicking::catch_unwind::do_call at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:581:40 62: std::panicking::catch_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:544:19 63: std::panic::catch_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panic.rs:359:14 64: std::thread::lifecycle::spawn_unchecked::{{closure}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/lifecycle.rs:89:26 65: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 66: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/alloc/src/boxed.rs:2206:9 67: std::sys::thread::unix::Thread::new::thread_start at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/sys/thread/unix.rs:127:17 68: _thrp_setup 69: <unknown> stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: unwrap<(), anyhow::Error> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1233:23 4: {async_block#6} at /work/oxidecomputer/omicron/sp-sim/src/gimlet.rs:429:62 5: poll<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 6: <tokio::runtime::task::core::Core<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/core.rs:375:24 7: with_mut<tokio::runtime::task::core::Stage<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/loom/std/unsafe_cell.rs:16:9 8: poll<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/core.rs:364:30 9: tokio::runtime::task::harness::poll_future::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:535:30 10: <core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>::{closure#0}> as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panic/unwind_safe.rs:274:9 11: do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:581:40 12: __rust_try 13: catch_unwind<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:544:19 14: catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<sp_sim::gimlet::{impl#2}::spawn::{async_fn#0}::{async_block_env#6}, alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panic.rs:359:14 15: tokio::runtime::task::harness::poll_future::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:523:18 16: <tokio::runtime::task::harness::Harness<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:210:27 17: <tokio::runtime::task::harness::Harness<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/harness.rs:155:20 18: tokio::runtime::task::raw::poll::<core::pin::Pin<alloc::boxed::Box<<sp_sim::gimlet::Gimlet>::spawn::{closure#0}::{closure#6}>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/raw.rs:337:13 19: <tokio::runtime::task::raw::RawTask>::poll at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/raw.rs:267:18 20: <tokio::runtime::task::LocalNotified<alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle>>>::run at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/task/mod.rs:515:13 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:825:30 22: with_budget<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 23: budget<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 24: {closure#0}<(), tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure_env#1}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:373:43 25: <tokio::runtime::scheduler::current_thread::Context>::enter::<(), <tokio::runtime::scheduler::current_thread::Context>::run_task<(), <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1}>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 26: <tokio::runtime::scheduler::current_thread::Context>::run_task::<(), <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#1}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:373:28 27: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:821:43 28: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 29: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 30: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 31: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 32: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 33: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 34: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 35: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 36: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 37: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 38: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 39: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 40: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 41: test_affinity_group_usage at ./tests/integration_tests/affinity.rs:448:1 42: test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} at ./tests/integration_tests/affinity.rs:449:72 43: <test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 44: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. thread 'integration_tests::affinity::test_affinity_group_usage' (2) panicked at gateway-test-utils/src/setup.rs:261:6: called `Result::unwrap()` on an `Err` value: TimedOut(30.001472634s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: unwrap<(), omicron_test_utils::dev::poll::Error<core::convert::Infallible>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1233:23 4: {async_fn#0} at /work/oxidecomputer/omicron/gateway-test-utils/src/setup.rs:261:6 5: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:417:10 6: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 7: poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/time/timeout.rs:202:42 8: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:233:59 9: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/starter.rs:1691:14 10: {async_fn#0}<omicron_nexus::Server> at ./test-utils/src/nexus_test.rs:94:10 11: {async_block#0} at ./tests/integration_tests/affinity.rs:448:1 12: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 13: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 15: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 16: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 17: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 18: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 19: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 20: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 21: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 22: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 23: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 24: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 25: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 26: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 27: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 28: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 29: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 30: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 31: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 32: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 33: test_affinity_group_usage at ./tests/integration_tests/affinity.rs:448:1 34: test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} at ./tests/integration_tests/affinity.rs:449:72 35: <test_all::integration_tests::affinity::test_affinity_group_usage::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 36: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp6VTNZA" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp6VTNZA/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 17245) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.1-clickhouse-IG1AmG) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-b1323e5628efcde2-test_affinity_group_usage.17207.1-clickhouse-IG1AmG: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpTba8LB WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpALb69d