(stdout not captured) (stderr not captured) (stdout not captured) (stderr not captured) thread 'integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume' panicked at nexus/tests/integration_tests/crucible_replacements.rs:1074:5: assertion failed: match last_background_task.last { LastResult::Completed(last_result_completed) => { match serde_json::from_value::<RegionReplacementDriverStatus>(last_result_completed.details) { Err(e) => { eprintln!("{e}"); false } Ok(v) => !v.drive_invoked_ok.is_empty(), } } _ => { false } } stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:697:5 1: core::panicking::panic_fmt at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:75:14 2: core::panicking::panic at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:145:5 3: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:1074:5 4: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:730:1 5: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 6: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 7: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:54 8: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:167:5 9: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:133:5 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:25 11: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:432:19 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:741:36 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:68 14: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/scoped.rs:40:9 15: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:26 16: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:315:12 17: std::thread::local::LocalKey<T>::with at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:279:15 18: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:9 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:27 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:729:19 21: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:200:28 22: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/runtime.rs:65:16 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:188:9 24: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:356:47 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:330:13 26: test_racing_replacements_for_soft_deleted_disk_volume at ./tests/integration_tests/crucible_replacements.rs:730:1 27: test_all::integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume::{{closure}} at ./tests/integration_tests/crucible_replacements.rs:733:2 28: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 29: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdUPk8p" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdUPk8p/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 22403) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.1-clickhouse-7JtTr1) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.1-clickhouse-7JtTr1: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpGfXlSr WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpWtIrUn WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpVmaIEs WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpOzOVgp running 1 test test integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume has been running for over 60 seconds test integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume ... FAILED failures: failures: integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 452 filtered out; finished in 74.71s log file: /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.0.log note: configured to log to "/var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.0.log" DB URL: postgresql://root@[::1]:62634/omicron?sslmode=disable DB address: [::1]:62634 log file: /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.2.log note: configured to log to "/var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.2.log" log file: /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.3.log note: configured to log to "/var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.3.log" old_region_id: Some(Region { identity: RegionIdentity { id: 77b77b81-81ae-40a1-bf0e-57db31e8143a, time_created: 2025-08-07T22:46:11.598723Z, time_modified: 2025-08-07T22:46:11.598723Z }, dataset_id: 149d20c7-9f4c-4024-be94-7855c1d3e364 (dataset), volume_id: 20f3c7c2-86c9-449f-b342-7bd173f625e5 (volume), block_size: ByteCount(ByteCount(512)), blocks_per_extent: 131072, extent_count: 16, port: Some(SqlU16(4000)), read_only: false, deleting: false, reservation_percent: TwentyFive }) new region id: None thread 'integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume' panicked at nexus/tests/integration_tests/crucible_replacements.rs:1074:5: assertion failed: match last_background_task.last { LastResult::Completed(last_result_completed) => { match serde_json::from_value::<RegionReplacementDriverStatus>(last_result_completed.details) { Err(e) => { eprintln!("{e}"); false } Ok(v) => !v.drive_invoked_ok.is_empty(), } } _ => { false } } stack backtrace: 0: __rustc::rust_begin_unwind at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/panicking.rs:697:5 1: core::panicking::panic_fmt at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:75:14 2: core::panicking::panic at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/panicking.rs:145:5 3: {async_fn#0} at ./tests/integration_tests/crucible_replacements.rs:1074:5 4: {async_block#0} at ./tests/integration_tests/crucible_replacements.rs:730:1 5: poll<&mut dyn core::future::future::Future<Output=()>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 6: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/future/future.rs:124:9 7: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:54 8: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:167:5 9: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/task/coop/mod.rs:133:5 10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:742:25 11: tokio::runtime::scheduler::current_thread::Context::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:432:19 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:741:36 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:68 14: tokio::runtime::context::scoped::Scoped<T>::set at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/scoped.rs:40:9 15: tokio::runtime::context::set_scheduler::{{closure}} at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:26 16: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)> at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:315:12 17: std::thread::local::LocalKey<T>::with at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/std/src/thread/local.rs:279:15 18: tokio::runtime::context::set_scheduler at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context.rs:176:9 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:829:27 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:729:19 21: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:200:28 22: tokio::runtime::context::runtime::enter_runtime at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/context/runtime.rs:65:16 23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/scheduler/current_thread/mod.rs:188:9 24: tokio::runtime::runtime::Runtime::block_on_inner at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:356:47 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.0/src/runtime/runtime.rs:330:13 26: test_racing_replacements_for_soft_deleted_disk_volume at ./tests/integration_tests/crucible_replacements.rs:730:1 27: test_all::integration_tests::crucible_replacements::test_racing_replacements_for_soft_deleted_disk_volume::{{closure}} at ./tests/integration_tests/crucible_replacements.rs:733:2 28: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 29: core::ops::function::FnOnce::call_once at /rustc/6b00bc3880198600130e1cf62b8f8a93494488cc/library/core/src/ops/function.rs:250:5 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace. WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpdUPk8p" If you would like to access the database for debugging, run the following: # Run the database cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpdUPk8p/data" # Access the database. Note the port may change if you run multiple databases. cockroach sql --host=localhost:32221 --insecure WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 22403) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.1-clickhouse-7JtTr1) failed to clean up ClickHouse data dir: - /var/tmp/omicron_tmp/test_all-a7445ac8ad43db82-test_racing_replacements_for_soft_deleted_disk_volume.22394.1-clickhouse-7JtTr1: File exists (os error 17) WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpGfXlSr WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmpWtIrUn WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpVmaIEs WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked) WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpOzOVgp