(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (171959) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.019496466s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpuuf60L" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpuuf60L/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 172768) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.1-clickhouse-IlmKXu) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQyuLiX WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpgTz6tB WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpJlAh7w WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpaWg2iK thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (239126) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.015047634s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpwiRnMo" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpwiRnMo/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 239172) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.1-clickhouse-fQFY0X) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp4lfOqq WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpfNaO0D WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpQpHWao WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp5MBVEv running 1 test test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done has been running for over 60 seconds test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done ... FAILED failures: failures: integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 276.53s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.0.log" DB URL: postgresql://root@[::1]:37137/omicron?sslmode=disable DB address: [::1]:37137 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.3.log" thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (239126) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.015047634s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpwiRnMo" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpwiRnMo/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 239172) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.239125.1-clickhouse-fQFY0X) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp4lfOqq WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpfNaO0D WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpQpHWao WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp5MBVEv thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (304120) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.015674947s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpq7sdtB" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpq7sdtB/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 304714) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.1-clickhouse-yDJ80i) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpGtnYTg WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp7HUXnz WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpA4oh9g WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpEgyJBj running 1 test test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done has been running for over 60 seconds test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done ... FAILED failures: failures: integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 279.28s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.0.log" DB URL: postgresql://root@[::1]:35207/omicron?sslmode=disable DB address: [::1]:35207 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.3.log" thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (304120) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.015674947s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpq7sdtB" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpq7sdtB/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 304714) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.304119.1-clickhouse-yDJ80i) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpGtnYTg WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp7HUXnz WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpA4oh9g WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpEgyJBj running 1 test test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done has been running for over 60 seconds test integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done ... FAILED failures: failures: integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 280.12s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.0.log" DB URL: postgresql://root@[::1]:43829/omicron?sslmode=disable DB address: [::1]:43829 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.3.log" thread 'integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done' (171959) panicked at nexus/tests/integration_tests/crucible_replacements.rs:486:14: request transitioned to expected state: TimedOut(260.019496466s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:486:14 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:657:14 6: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:746:59 7: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:725:1 8: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 11: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 12: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 14: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 15: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 16: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 17: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 19: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 20: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 21: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 23: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 24: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 25: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 27: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 28: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 29: test_delete_volume_region_replacement_state_replacement_done at ./tests/integration_tests/crucible_replacements.rs:725:1 30: test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:728:2 31: <test_all::integration_tests::crucible_replacements::test_delete_volume_region_replacement_state_replacement_done::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 32: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpuuf60L" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpuuf60L/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 172768) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_delete_volume_region_replacement_state_replacement_done.171958.1-clickhouse-IlmKXu) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQyuLiX WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpgTz6tB WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpJlAh7w WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpaWg2iK thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (178281) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.590087693s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp4sTOlx" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp4sTOlx/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 178443) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.1-clickhouse-L1J3UH) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpXfXNpD WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp1I0Mo6 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpPUZXok WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpO1hrsL thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (245560) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.780140309s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTPyhiP" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTPyhiP/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 245642) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.1-clickhouse-n9wJg4) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpU3HwIH WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpe6zhZX WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpvtcMBD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp5Z6jNw running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 279.86s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.0.log" DB URL: postgresql://root@[::1]:40967/omicron?sslmode=disable DB address: [::1]:40967 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (245560) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.780140309s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTPyhiP" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTPyhiP/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 245642) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.245559.1-clickhouse-n9wJg4) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpU3HwIH WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpe6zhZX WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpvtcMBD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp5Z6jNw thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (313779) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.309796033s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpaVDHbH" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpaVDHbH/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 314258) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.1-clickhouse-PPdErj) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpbW6Whu WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpBa1FQW WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpS1XbAD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpTkZjA7 running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 282.38s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.0.log" DB URL: postgresql://root@[::1]:39679/omicron?sslmode=disable DB address: [::1]:39679 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (313779) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.309796033s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpaVDHbH" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpaVDHbH/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 314258) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.313778.1-clickhouse-PPdErj) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpbW6Whu WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpBa1FQW WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpS1XbAD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpTkZjA7 running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 281.44s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.0.log" DB URL: postgresql://root@[::1]:32911/omicron?sslmode=disable DB address: [::1]:32911 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity' (178281) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.590087693s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2238:65 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2147:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity at ./tests/integration_tests/crucible_replacements.rs:2147:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2150:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp4sTOlx" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp4sTOlx/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 178443) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity.178280.1-clickhouse-L1J3UH) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpXfXNpD WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp1I0Mo6 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpPUZXok WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpO1hrsL thread 'integration_tests::crucible_replacements::test_replacement_sanity' (179290) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.294152763s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp0KswSX" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp0KswSX/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 179459) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.1-clickhouse-JJmTE0) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmphuomDC WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpHerSCR WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpfjlsfQ WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpzMPizy thread 'integration_tests::crucible_replacements::test_replacement_sanity' (246873) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.933075206s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp7RCFat" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp7RCFat/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 247476) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.1-clickhouse-LyuTen) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpe7GSXG WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmptpMyZ1 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpaF7WyR WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpRGHbi0 running 1 test test integration_tests::crucible_replacements::test_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 279.95s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.0.log" DB URL: postgresql://root@[::1]:38227/omicron?sslmode=disable DB address: [::1]:38227 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.3.log" thread 'integration_tests::crucible_replacements::test_replacement_sanity' (246873) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.933075206s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp7RCFat" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp7RCFat/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 247476) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.246872.1-clickhouse-LyuTen) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpe7GSXG WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmptpMyZ1 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpaF7WyR WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpRGHbi0 thread 'integration_tests::crucible_replacements::test_replacement_sanity' (315798) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.065514078s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpgfD9c9" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpgfD9c9/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 316038) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.1-clickhouse-kzeMQs) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp5AYtkj WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp1NNzBN WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpP4rGQN WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpdczCf4 running 1 test test integration_tests::crucible_replacements::test_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 279.54s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.0.log" DB URL: postgresql://root@[::1]:40911/omicron?sslmode=disable DB address: [::1]:40911 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.3.log" thread 'integration_tests::crucible_replacements::test_replacement_sanity' (315798) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.065514078s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpgfD9c9" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpgfD9c9/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 316038) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.315711.1-clickhouse-kzeMQs) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp5AYtkj WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp1NNzBN WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpP4rGQN WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpdczCf4 running 1 test test integration_tests::crucible_replacements::test_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 279.67s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.0.log" DB URL: postgresql://root@[::1]:33123/omicron?sslmode=disable DB address: [::1]:33123 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.3.log" thread 'integration_tests::crucible_replacements::test_replacement_sanity' (179290) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.294152763s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2127:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2048:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2048:1 29: test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2049:70 30: <test_all::integration_tests::crucible_replacements::test_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp0KswSX" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp0KswSX/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 179459) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_replacement_sanity.179274.1-clickhouse-JJmTE0) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmphuomDC WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpHerSCR WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpfjlsfQ WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpzMPizy thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (178950) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.199284361s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTFzJW0" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTFzJW0/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 178996) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.1-clickhouse-xnvhtX) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpIiw3fJ WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpjUodDT WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpNZv475 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpyJbgLl thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (246730) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.482540229s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpcj1PKb" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpcj1PKb/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 246904) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.1-clickhouse-IX07s0) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDpNJRi WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpyOr1wA WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp62rfBh WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpU2E2Qe running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 280.78s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.0.log" DB URL: postgresql://root@[::1]:39853/omicron?sslmode=disable DB address: [::1]:39853 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (246730) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.482540229s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpcj1PKb" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpcj1PKb/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 246904) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.246729.1-clickhouse-IX07s0) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpDpNJRi WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpyOr1wA WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp62rfBh WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpU2E2Qe thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (315799) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.389385946s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp9VXih2" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp9VXih2/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 316041) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.1-clickhouse-QNfK2s) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpgw1gN1 WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpBYcTaG WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp4jGDjD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpEJ6ovI running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 282.07s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.0.log" DB URL: postgresql://root@[::1]:39801/omicron?sslmode=disable DB address: [::1]:39801 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (315799) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.389385946s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmp9VXih2" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmp9VXih2/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 316041) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.315795.1-clickhouse-QNfK2s) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpgw1gN1 WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpBYcTaG WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp4jGDjD WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpEJ6ovI running 1 test test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 has been running for over 60 seconds test integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 ... FAILED failures: failures: integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2 test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 280.64s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.0.log" DB URL: postgresql://root@[::1]:41917/omicron?sslmode=disable DB address: [::1]:41917 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.3.log" thread 'integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2' (178950) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.199284361s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2371:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2273:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_region_replacement_triple_sanity_2 at ./tests/integration_tests/crucible_replacements.rs:2273:1 29: test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2276:2 30: <test_all::integration_tests::crucible_replacements::test_region_replacement_triple_sanity_2::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTFzJW0" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTFzJW0/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 178996) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_region_replacement_triple_sanity_2.178949.1-clickhouse-xnvhtX) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpIiw3fJ WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpjUodDT WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpNZv475 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpyJbgLl thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (177680) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.631134746s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTaj993" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTaj993/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 177741) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.1-clickhouse-VXXsq1) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmphKtjfo WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpHezJix WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp9FbQVn WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpMbND5R thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (248399) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.237245975s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpLbCsMM" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpLbCsMM/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 248849) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.1-clickhouse-ntoe8q) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpdu4aJJ WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpiyfvZm WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpRmBvJS WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUvFaOt running 1 test test integration_tests::crucible_replacements::test_read_only_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_read_only_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_read_only_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 292.46s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.0.log" DB URL: postgresql://root@[::1]:46345/omicron?sslmode=disable DB address: [::1]:46345 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.3.log" thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (248399) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.237245975s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpLbCsMM" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpLbCsMM/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 248849) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.248396.1-clickhouse-ntoe8q) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpdu4aJJ WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpiyfvZm WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpRmBvJS WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUvFaOt thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (320917) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.042300705s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdnfOaG" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdnfOaG/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 321054) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.1-clickhouse-laQpx6) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpt4awLt WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQHMRLz WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpSq0XEZ WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp3VNuyT running 1 test test integration_tests::crucible_replacements::test_read_only_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_read_only_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_read_only_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 291.73s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.0.log" DB URL: postgresql://root@[::1]:34165/omicron?sslmode=disable DB address: [::1]:34165 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.3.log" thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (320917) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.042300705s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdnfOaG" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdnfOaG/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 321054) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.320908.1-clickhouse-laQpx6) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpt4awLt WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQHMRLz WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpSq0XEZ WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp3VNuyT running 1 test test integration_tests::crucible_replacements::test_read_only_replacement_sanity has been running for over 60 seconds test integration_tests::crucible_replacements::test_read_only_replacement_sanity ... FAILED failures: failures: integration_tests::crucible_replacements::test_read_only_replacement_sanity test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 560 filtered out; finished in 292.52s log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.0.log" DB URL: postgresql://root@[::1]:40747/omicron?sslmode=disable DB address: [::1]:40747 log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.2.log" log file: /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.3.log" thread 'integration_tests::crucible_replacements::test_read_only_replacement_sanity' (177680) panicked at nexus/tests/integration_tests/crucible_replacements.rs:208:6: all replacements finished: TimedOut(260.631134746s) stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::result::unwrap_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1867:5 3: <core::result::Result<(), omicron_test_utils::dev::poll::Error<()>>>::expect at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/result.rs:1185:23 4: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:208:6 5: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:2665:61 6: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:2543:1 7: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25 13: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44 15: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68 16: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12 19: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)> at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27 22: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33 24: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9 26: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18 28: test_read_only_replacement_sanity at ./tests/integration_tests/crucible_replacements.rs:2543:1 29: test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} at ./tests/integration_tests/crucible_replacements.rs:2546:2 30: <test_all::integration_tests::crucible_replacements::test_read_only_replacement_sanity::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 31: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpTaj993" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpTaj993/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 177741) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-bc84eedbcabfcefd-test_read_only_replacement_sanity.177679.1-clickhouse-VXXsq1) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmphKtjfo WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpHezJix WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmp9FbQVn WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpMbND5R thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (371764) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (371899) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. running 1 test latest url pointer: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-latest.json" latest_name: "crucible-pantry-2.0.0-9dbf85.json" raw_url: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-2.0.0-9dbf85.json" test sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real ... FAILED failures: failures: sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 56 filtered out; finished in 1.03s thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (371899) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (372083) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. running 1 test latest url pointer: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-latest.json" latest_name: "crucible-pantry-2.0.0-9dbf85.json" raw_url: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-2.0.0-9dbf85.json" test sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real ... FAILED failures: failures: sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 56 filtered out; finished in 1.04s thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (372083) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. running 1 test latest url pointer: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-latest.json" latest_name: "crucible-pantry-2.0.0-9dbf85.json" raw_url: "https://raw.githubusercontent.com/oxidecomputer/crucible/fefccc0623038371dbded0458f0645bee6b141cb/openapi/crucible-pantry/crucible-pantry-2.0.0-9dbf85.json" test sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real ... FAILED failures: failures: sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 56 filtered out; finished in 2.52s thread 'sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real' (371764) panicked at sled-agent/src/sim/http_entrypoints_pantry.rs:466:17: assertion `left == right` failed left: 3 right: 4 stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5 1: core::panicking::panic_fmt at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14 2: core::panicking::assert_failed_inner at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:439:17 3: core::panicking::assert_failed at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:394:5 4: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:466:17 5: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 6: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 7: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 8: compare_json_values at ./src/sim/http_entrypoints_pantry.rs:495:21 9: test_simulated_api_matches_real at ./src/sim/http_entrypoints_pantry.rs:454:9 10: omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} at ./src/sim/http_entrypoints_pantry.rs:438:41 11: <omicron_sled_agent::sim::http_entrypoints_pantry::tests::test_simulated_api_matches_real::{closure#0} as core::ops::function::FnOnce<()>>::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 12: core::ops::function::FnOnce::call_once at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.