(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'app::sagas::disk_create::test::test_action_failure_can_unwind' (2) panicked at nexus/src/app/sagas/disk_create.rs:1391:9: assertion failed: no_disk_records_exist(datastore).await stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: core::panicking::panic at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:145:5 3: {async_fn#0} at ./src/app/sagas/disk_create.rs:1391:9 4: {async_block#0} at ./src/app/sagas/disk_create.rs:1454:59 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 6: {async_fn#0}<omicron_nexus::app::sagas::disk_create::SagaDiskCreate, omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{async_block#0}::test_action_failure_can_unwind::{async_fn#0}::{closure_env#0}, omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{async_block#0}::test_action_failure_can_unwind::{async_fn#0}::{closure_env#1}> at ./src/app/sagas/test_helpers.rs:639:22 7: {async_fn#0} at ./src/app/sagas/disk_create.rs:1459:10 8: {async_block#0} at ./src/app/sagas/disk_create.rs:1432:5 9: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 10: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 11: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 12: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 13: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 15: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 16: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 17: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 18: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 19: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 20: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 21: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 22: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 23: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 24: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 25: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 26: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 28: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 29: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 30: test_action_failure_can_unwind at ./src/app/sagas/disk_create.rs:1432:5 31: omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{{closure}} at ./src/app/sagas/disk_create.rs:1435:6 32: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 33: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpsuZLC5" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpsuZLC5/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 13008) and a temporary directory leaked, /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.1-clickhouse-Zy6QMk) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.1-clickhouse-Zy6QMk: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQ7uoLP WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmphORosd running 1 test test app::sagas::disk_create::test::test_action_failure_can_unwind ... FAILED failures: failures: app::sagas::disk_create::test::test_action_failure_can_unwind test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 252 filtered out; finished in 42.68s log file: /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.0.log note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.0.log" DB URL: postgresql://root@[::1]:54955/omicron?sslmode=disable DB address: [::1]:54955 log file: /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.2.log note: configured to log to "/var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.2.log" thread 'app::sagas::disk_create::test::test_action_failure_can_unwind' (2) panicked at nexus/src/app/sagas/disk_create.rs:1391:9: assertion failed: no_disk_records_exist(datastore).await stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: core::panicking::panic at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:145:5 3: {async_fn#0} at ./src/app/sagas/disk_create.rs:1391:9 4: {async_block#0} at ./src/app/sagas/disk_create.rs:1454:59 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 6: {async_fn#0}<omicron_nexus::app::sagas::disk_create::SagaDiskCreate, omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{async_block#0}::test_action_failure_can_unwind::{async_fn#0}::{closure_env#0}, omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{async_block#0}::test_action_failure_can_unwind::{async_fn#0}::{closure_env#1}> at ./src/app/sagas/test_helpers.rs:639:22 7: {async_fn#0} at ./src/app/sagas/disk_create.rs:1459:10 8: {async_block#0} at ./src/app/sagas/disk_create.rs:1432:5 9: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 10: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 11: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 12: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 13: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 15: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 16: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 17: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 18: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 19: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 20: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 21: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 22: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 23: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 24: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 25: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 26: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 28: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 29: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 30: test_action_failure_can_unwind at ./src/app/sagas/disk_create.rs:1432:5 31: omicron_nexus::app::sagas::disk_create::test::test_action_failure_can_unwind::{{closure}} at ./src/app/sagas/disk_create.rs:1435:6 32: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 33: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpsuZLC5" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpsuZLC5/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 13008) and a temporary directory leaked, /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.1-clickhouse-Zy6QMk) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/omicron_nexus-27e683fd83af0746-test_action_failure_can_unwind.12955.1-clickhouse-Zy6QMk: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpQ7uoLP WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmphORosd thread 'integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete' (2) panicked at nexus/tests/integration_tests/disks.rs:1459:6: unexpected failure deleting project: expected status code 204 No Content, found 400 Bad Request Stack backtrace: 0: anyhow::error::<impl anyhow::Error>::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 1: {async_fn#0} at ./test-utils/src/http_testing.rs:361:13 2: {async_fn#0} at ./test-utils/src/http_testing.rs:661:40 3: {async_fn#0} at ./tests/integration_tests/disks.rs:1458:6 4: {async_block#0} at ./tests/integration_tests/disks.rs:1323:1 5: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 6: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 7: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 8: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 9: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 11: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 14: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 15: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 16: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 17: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 18: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 21: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 22: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 24: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 26: test_disk_virtual_provisioning_collection_failed_delete at ./tests/integration_tests/disks.rs:1323:1 27: test_all::integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete::{{closure}} at ./tests/integration_tests/disks.rs:1326:2 28: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 29: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 30: test::__rust_begin_short_backtrace at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:663:18 31: test::run_test_in_process::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:686:74 32: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panic/unwind_safe.rs:274:9 33: std::panicking::catch_unwind::do_call at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:590:40 34: std::panicking::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:553:19 35: std::panic::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panic.rs:359:14 36: test::run_test_in_process at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:686:27 37: test::run_test::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:607:43 38: test::run_test::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:637:41 39: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/sys/backtrace.rs:158:18 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/mod.rs:559:17 41: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panic/unwind_safe.rs:274:9 42: std::panicking::catch_unwind::do_call at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:590:40 43: std::panicking::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:553:19 44: std::panic::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panic.rs:359:14 45: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/mod.rs:557:30 46: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 47: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/alloc/src/boxed.rs:1985:9 48: std::sys::thread::unix::Thread::new::thread_start at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/sys/thread/unix.rs:126:17 49: _thrp_setup 50: <unknown> stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/result.rs:1855:5 3: core::result::Result<T,E>::expect at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/result.rs:1178:23 4: {async_fn#0} at ./tests/integration_tests/disks.rs:1459:6 5: {async_block#0} at ./tests/integration_tests/disks.rs:1323:1 6: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 7: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 8: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 9: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 10: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 11: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 12: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 15: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 16: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 17: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 18: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 19: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 22: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 23: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 24: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 25: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 27: test_disk_virtual_provisioning_collection_failed_delete at ./tests/integration_tests/disks.rs:1323:1 28: test_all::integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete::{{closure}} at ./tests/integration_tests/disks.rs:1326:2 29: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 30: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpmiwMLK" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpmiwMLK/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 28287) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.1-clickhouse-iEgXuS) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.1-clickhouse-iEgXuS: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpnPsrOU WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpOgVG8l running 1 test test integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete ... FAILED failures: failures: integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 555 filtered out; finished in 45.86s log file: /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.0.log" DB URL: postgresql://root@[::1]:61442/omicron?sslmode=disable DB address: [::1]:61442 log file: /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.2.log" thread 'integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete' (2) panicked at nexus/tests/integration_tests/disks.rs:1459:6: unexpected failure deleting project: expected status code 204 No Content, found 400 Bad Request Stack backtrace: 0: anyhow::error::<impl anyhow::Error>::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 1: {async_fn#0} at ./test-utils/src/http_testing.rs:361:13 2: {async_fn#0} at ./test-utils/src/http_testing.rs:661:40 3: {async_fn#0} at ./tests/integration_tests/disks.rs:1458:6 4: {async_block#0} at ./tests/integration_tests/disks.rs:1323:1 5: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 6: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 7: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 8: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 9: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 11: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 14: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 15: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 16: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 17: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 18: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 21: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 22: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 24: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 26: test_disk_virtual_provisioning_collection_failed_delete at ./tests/integration_tests/disks.rs:1323:1 27: test_all::integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete::{{closure}} at ./tests/integration_tests/disks.rs:1326:2 28: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 29: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 30: test::__rust_begin_short_backtrace at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:663:18 31: test::run_test_in_process::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:686:74 32: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panic/unwind_safe.rs:274:9 33: std::panicking::catch_unwind::do_call at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:590:40 34: std::panicking::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:553:19 35: std::panic::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panic.rs:359:14 36: test::run_test_in_process at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:686:27 37: test::run_test::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:607:43 38: test::run_test::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/test/src/lib.rs:637:41 39: std::sys::backtrace::__rust_begin_short_backtrace at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/sys/backtrace.rs:158:18 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/mod.rs:559:17 41: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panic/unwind_safe.rs:274:9 42: std::panicking::catch_unwind::do_call at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:590:40 43: std::panicking::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:553:19 44: std::panic::catch_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panic.rs:359:14 45: std::thread::Builder::spawn_unchecked_::{{closure}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/mod.rs:557:30 46: core::ops::function::FnOnce::call_once{{vtable.shim}} at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 47: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/alloc/src/boxed.rs:1985:9 48: std::sys::thread::unix::Thread::new::thread_start at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/sys/thread/unix.rs:126:17 49: _thrp_setup 50: <unknown> stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: core::result::unwrap_failed at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/result.rs:1855:5 3: core::result::Result<T,E>::expect at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/result.rs:1178:23 4: {async_fn#0} at ./tests/integration_tests/disks.rs:1459:6 5: {async_block#0} at ./tests/integration_tests/disks.rs:1323:1 6: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 7: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 8: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 9: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 10: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 11: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 12: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 15: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 16: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 17: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 18: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 19: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 22: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 23: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 24: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 25: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 27: test_disk_virtual_provisioning_collection_failed_delete at ./tests/integration_tests/disks.rs:1323:1 28: test_all::integration_tests::disks::test_disk_virtual_provisioning_collection_failed_delete::{{closure}} at ./tests/integration_tests/disks.rs:1326:2 29: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 30: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpmiwMLK" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpmiwMLK/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 28287) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.1-clickhouse-iEgXuS) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-test_disk_virtual_provisioning_collection_failed_delete.28278.1-clickhouse-iEgXuS: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpnPsrOU WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpOgVG8l thread 'integration_tests::schema::nexus_applies_update_on_boot' (2) panicked at nexus/tests/integration_tests/schema.rs:411:5: Nexus should have started stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: {async_block#0} at ./tests/integration_tests/schema.rs:411:5 3: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 4: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 5: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 6: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 7: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 8: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 9: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 11: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 12: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 13: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 14: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 15: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 16: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 17: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 18: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 19: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 20: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 21: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 22: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 24: nexus_applies_update_on_boot at ./tests/integration_tests/schema.rs:425:29 25: test_all::integration_tests::schema::nexus_applies_update_on_boot::{{closure}} at ./tests/integration_tests/schema.rs:381:40 26: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 27: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdhju5N" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdhju5N/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp85X61x WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpLUvjA1 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUeLa9u WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUeNTLE running 1 test test integration_tests::schema::nexus_applies_update_on_boot has been running for over 60 seconds test integration_tests::schema::nexus_applies_update_on_boot ... FAILED failures: failures: integration_tests::schema::nexus_applies_update_on_boot test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 555 filtered out; finished in 123.20s log file: /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.0.log" DB URL: postgresql://root@[::1]:45457/omicron?sslmode=disable DB address: [::1]:45457 log file: /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.1.log note: configured to log to "/var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.1.log" log file: /var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-46ec180f6e3b8cef-nexus_applies_update_on_boot.3373.2.log" thread 'integration_tests::schema::nexus_applies_update_on_boot' (2) panicked at nexus/tests/integration_tests/schema.rs:411:5: Nexus should have started stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5 1: core::panicking::panic_fmt at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14 2: {async_block#0} at ./tests/integration_tests/schema.rs:411:5 3: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 4: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9 5: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70 6: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5 7: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5 8: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25 9: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44 11: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68 12: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9 13: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38 14: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12 15: std::thread::local::LocalKey<T>::with at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20 16: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17 17: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27 18: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24 19: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33 20: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16 21: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9 22: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18 24: nexus_applies_update_on_boot at ./tests/integration_tests/schema.rs:425:29 25: test_all::integration_tests::schema::nexus_applies_update_on_boot::{{closure}} at ./tests/integration_tests/schema.rs:381:40 26: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 27: core::ops::function::FnOnce::call_once at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdhju5N" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdhju5N/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmp85X61x WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpLUvjA1 WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUeLa9u WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpUeNTLE